problem.cc
Go to the documentation of this file.
1// LIC// ====================================================================
2// LIC// This file forms part of oomph-lib, the object-oriented,
3// LIC// multi-physics finite-element library, available
4// LIC// at http://www.oomph-lib.org.
5// LIC//
6// LIC// Copyright (C) 2006-2023 Matthias Heil and Andrew Hazel
7// LIC//
8// LIC// This library is free software; you can redistribute it and/or
9// LIC// modify it under the terms of the GNU Lesser General Public
10// LIC// License as published by the Free Software Foundation; either
11// LIC// version 2.1 of the License, or (at your option) any later version.
12// LIC//
13// LIC// This library is distributed in the hope that it will be useful,
14// LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// LIC// Lesser General Public License for more details.
17// LIC//
18// LIC// You should have received a copy of the GNU Lesser General Public
19// LIC// License along with this library; if not, write to the Free Software
20// LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21// LIC// 02110-1301 USA.
22// LIC//
23// LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
24// LIC//
25// LIC//====================================================================
26
27#ifdef OOMPH_HAS_MPI
28#include "mpi.h"
29#endif
30
31#include <list>
32#include <algorithm>
33#include <string>
34
35#include "oomph_utilities.h"
36#include "problem.h"
37#include "timesteppers.h"
40#include "refineable_mesh.h"
41#include "triangle_mesh.h"
42#include "linear_solver.h"
43#include "eigen_solver.h"
44#include "assembly_handler.h"
45#include "dg_elements.h"
46#include "partitioning.h"
47#include "spines.h"
48
49// Include to fill in additional_setup_shared_node_scheme() function
51
52
53namespace oomph
54{
55 /// ///////////////////////////////////////////////////////////////
56 // Non-inline functions for the problem class
57 /// ///////////////////////////////////////////////////////////////
58
59 //=================================================================
60 /// The continuation timestepper object
61 //=================================================================
62 ContinuationStorageScheme Problem::Continuation_time_stepper;
63
64 //================================================================
65 /// Constructor: Allocate space for one time stepper
66 /// and set all pointers to NULL and set defaults for all
67 /// parameters.
68 //===============================================================
70 : Mesh_pt(0),
71 Time_pt(0),
72 Explicit_time_stepper_pt(0),
73 Saved_dof_pt(0),
74 Default_set_initial_condition_called(false),
75 Use_globally_convergent_newton_method(false),
76 Empty_actions_before_read_unstructured_meshes_has_been_called(false),
77 Empty_actions_after_read_unstructured_meshes_has_been_called(false),
78 Store_local_dof_pt_in_elements(false),
79 Calculate_hessian_products_analytic(false),
81 Doc_imbalance_in_parallel_assembly(false),
82 Use_default_partition_in_load_balance(false),
83 Must_recompute_load_balance_for_assembly(true),
84 Halo_scheme_pt(0),
85#endif
86 Relaxation_factor(1.0),
87 Newton_solver_tolerance(1.0e-8),
88 Max_newton_iterations(10),
89 Nnewton_iter_taken(0),
90 Max_residuals(10.0),
91 Time_adaptive_newton_crash_on_solve_fail(false),
92 Jacobian_reuse_is_enabled(false),
93 Jacobian_has_been_computed(false),
94 Problem_is_nonlinear(true),
95 Pause_at_end_of_sparse_assembly(false),
96 Doc_time_in_distribute(false),
97 Sparse_assembly_method(Perform_assembly_using_vectors_of_pairs),
98 Sparse_assemble_with_arrays_initial_allocation(400),
99 Sparse_assemble_with_arrays_allocation_increment(150),
100 Numerical_zero_for_sparse_assembly(0.0),
101 FD_step_used_in_get_hessian_vector_products(1.0e-8),
102 Mass_matrix_reuse_is_enabled(false),
103 Mass_matrix_has_been_computed(false),
104 Discontinuous_element_formulation(false),
105 Minimum_dt(1.0e-12),
106 Maximum_dt(1.0e12),
107 DTSF_max_increase(4.0),
108 DTSF_min_decrease(0.8),
109 Minimum_dt_but_still_proceed(-1.0),
110 Scale_arc_length(true),
111 Desired_proportion_of_arc_length(0.5),
112 Theta_squared(1.0),
113 Sign_of_jacobian(0),
114 Continuation_direction(1.0),
115 Parameter_derivative(1.0),
116 Parameter_current(0.0),
117 Use_continuation_timestepper(false),
118 Dof_derivative_offset(1),
119 Dof_current_offset(2),
120 Ds_current(0.0),
121 Desired_newton_iterations_ds(5),
122 Minimum_ds(1.0e-10),
123 Bifurcation_detection(false),
124 Bisect_to_find_bifurcation(false),
125 First_jacobian_sign_change(false),
126 Arc_length_step_taken(false),
127 Use_finite_differences_for_continuation_derivatives(false),
129 Dist_problem_matrix_distribution(Uniform_matrix_distribution),
130 Parallel_sparse_assemble_previous_allocation(0),
131 Problem_has_been_distributed(false),
132 Bypass_increase_in_dof_check_during_pruning(false),
133 Max_permitted_error_for_halo_check(1.0e-14),
134#endif
135 Shut_up_in_newton_solve(false),
136 Always_take_one_newton_step(false),
137 Timestep_reduction_factor_after_nonconvergence(0.5),
138 Keep_temporal_error_below_tolerance(true)
139 {
141
142 /// Setup terminate helper
144
145 // By default no submeshes:
146 Sub_mesh_pt.resize(0);
147 // No timesteppers
148 Time_stepper_pt.resize(0);
149
150 // Set the linear solvers, eigensolver and assembly handler
153
155
157
158 // setup the communicator
159#ifdef OOMPH_HAS_MPI
161 {
163 }
164 else
165 {
167 }
168#else
170#endif
171
172 // just create an empty linear algebra distribution for the
173 // DOFs
174 // this is setup when assign_eqn_numbers(...) is called.
176 }
177
178 //================================================================
179 /// Destructor to clean up memory
180 //================================================================
182 {
183 // Delete the memory assigned for the global time
184 // (it's created on the fly in Problem::add_time_stepper_pt()
185 // so we are entitled to delete it.
186 if (Time_pt != 0)
187 {
188 delete Time_pt;
189 Time_pt = 0;
190 }
191
192 // We're not using the default linear solver,
193 // somebody else must have built it, so that person
194 // must be in charge of killing it.
195 // We can safely delete the defaults, however
197
200 delete Communicator_pt;
201 delete Dof_distribution_pt;
202
203 // Delete any copies of the problem that have been created for
204 // use in adaptive bifurcation tracking.
205 // ALH: This will eventually go
206 unsigned n_copies = Copy_of_problem_pt.size();
207 for (unsigned c = 0; c < n_copies; c++)
208 {
209 delete Copy_of_problem_pt[c];
210 }
211
212 // if this problem has sub meshes then we must delete the Mesh_pt
213 if (Sub_mesh_pt.size() != 0)
214 {
216 delete Mesh_pt;
217 }
218
219 // Since we called the TerminateHelper setup function in the constructor,
220 // we need to delete anything that was dynamically allocated (as it's
221 // just a namespace and so doesn't have it's own destructor) in the function
223 }
224
225 //=================================================================
226 /// Setup the count vector that records how many elements contribute
227 /// to each degree of freedom. Returns the total number of elements
228 /// in the problem
229 //=================================================================
231 {
232 // Now set the element counter to have the current Dof distribution
234 // We need to use the halo scheme (assuming it has been setup)
235#ifdef OOMPH_HAS_MPI
237#endif
238
239 // Loop over the elements and count the entries
240 // and number of (non-halo) elements
241 const unsigned n_element = this->mesh_pt()->nelement();
242 unsigned n_non_halo_element_local = 0;
243 for (unsigned e = 0; e < n_element; e++)
244 {
246#ifdef OOMPH_HAS_MPI
247 // Ignore halo elements
248 if (!elem_pt->is_halo())
249 {
250#endif
251 // Increment the number of non halo elements
253 // Now count the number of times the element contributes to a value
254 // using the current assembly handler
255 unsigned n_var = this->Assembly_handler_pt->ndof(elem_pt);
256 for (unsigned n = 0; n < n_var; n++)
257 {
259 this->Assembly_handler_pt->eqn_number(elem_pt, n));
260 }
261#ifdef OOMPH_HAS_MPI
262 }
263#endif
264 }
265
266 // Storage for the total number of elements
267 unsigned Nelement = 0;
268
269 // Add together all the counts if we are in parallel
270#ifdef OOMPH_HAS_MPI
272
273 // If distributed, find the total number of elements in the problem
275 {
276 // Need to gather the total number of non halo elements
278 &Nelement,
279 1,
281 MPI_SUM,
282 this->communicator_pt()->mpi_comm());
283 }
284 // Otherwise the total number is the same on each processor
285 else
286#endif
287 {
288 Nelement = n_non_halo_element_local;
289 }
290
291 return Nelement;
292 }
293
294
295 //==================================================================
296 /// Build new LinearAlgebraDistribution. Note: you're in charge of
297 /// deleting it!
298 //==================================================================
301 {
302 // Find the number of rows
303 const unsigned nrow = this->ndof();
304
305#ifdef OOMPH_HAS_MPI
306
307 unsigned nproc = Communicator_pt->nproc();
308
309 // if problem is only one one processor assemble non-distributed
310 // distribution
311 if (nproc == 1)
312 {
314 }
315 // if the problem is not distributed then assemble the jacobian with
316 // a uniform distributed distribution
318 {
320 }
321 // otherwise the problem is a distributed problem
322 else
323 {
325 {
327
329 break;
330
332
334 break;
335
337
338 // Put in its own scope to avoid warnings about "local" variables
339 {
342 bool use_problem_dist = true;
343 for (unsigned p = 0; p < nproc; p++)
344 {
345 // hierher Andrew: what's the logic behind this?
346 if ((double)Dof_distribution_pt->nrow_local(p) >
347 ((double)uniform_dist_pt->nrow_local(p)) * 1.1)
348 {
349 use_problem_dist = false;
350 }
351 }
353 {
355 }
356 else
357 {
359 }
360 delete uniform_dist_pt;
361 }
362 break;
363
364 default:
365
366 std::ostringstream error_stream;
367 error_stream << "Never get here. Dist_problem_matrix_distribution = "
368 << Dist_problem_matrix_distribution << std::endl;
369 throw OomphLibError(error_stream.str(),
372 break;
373 }
374 }
375#else
377#endif
378 }
379
380
381#ifdef OOMPH_HAS_MPI
382
383 //==================================================================
384 /// Setup the halo scheme for the degrees of freedom
385 //==================================================================
387 {
388 // Find the number of elements stored on this processor
389 const unsigned n_element = this->mesh_pt()->nelement();
390
391 // Work out the all global equations to which this processor
392 // contributes
394 this->get_my_eqns(this->Assembly_handler_pt, 0, n_element - 1, my_eqns);
395
396 // Build the halo scheme, based on the equations to which this
397 // processor contributes
400
401 // Find pointers to all the halo dofs
402 // There may be more of these than required by my_eqns
403 //(but should not be less)
404 std::map<unsigned, double*> halo_data_pt;
405 this->get_all_halo_data(halo_data_pt);
406
407 // Now setup the Halo_dofs
409 }
410
411 //==================================================================
412 /// Distribute the problem without doc; report stats if required.
413 /// Returns actual partitioning used, e.g. for restart.
414 //==================================================================
416 {
417 // Set dummy doc paramemters
418 DocInfo doc_info;
419 doc_info.disable_doc();
420
421 // Set the sizes of the input and output vectors
422 unsigned n_element = mesh_pt()->nelement();
424
425 // Distribute and return partitioning
426 return distribute(element_partition, doc_info, report_stats);
427 }
428
429 //==================================================================
430 /// Distribute the problem according to specified partition.
431 /// If all entries in partitioning vector are zero we use METIS
432 /// to do the partitioning after all.
433 /// Returns actual partitioning used, e.g. for restart.
434 //==================================================================
437 {
438#ifdef PARANOID
439 bool has_non_zero_entry = false;
440 unsigned n = element_partition.size();
441 for (unsigned i = 0; i < n; i++)
442 {
443 if (element_partition[i] != 0)
444 {
445 has_non_zero_entry = true;
446 break;
447 }
448 }
450 {
451 std::ostringstream warn_message;
452 warn_message << "WARNING: All entries in specified partitioning vector \n"
453 << " are zero -- will ignore this and use METIS\n"
454 << " to perform the partitioning\n";
456 warn_message.str(), "Problem::distribute()", OOMPH_EXCEPTION_LOCATION);
457 }
458#endif
459 // Set dummy doc paramemters
460 DocInfo doc_info;
461 doc_info.disable_doc();
462
463 // Distribute and return partitioning
464 return distribute(element_partition, doc_info, report_stats);
465 }
466
467 //==================================================================
468 /// Distribute the problem and doc to specified DocInfo.
469 /// Returns actual partitioning used, e.g. for restart.
470 //==================================================================
472 const bool& report_stats)
473 {
474 // Set the sizes of the input and output vectors
475 unsigned n_element = mesh_pt()->nelement();
476
477 // Dummy input vector
479
480 // Distribute and return partitioning
481 return distribute(element_partition, doc_info, report_stats);
482 }
483
484 //==================================================================
485 /// Distribute the problem according to specified partition.
486 /// (If all entries in partitioning vector are zero we use METIS
487 /// to do the partitioning after all) and doc.
488 /// Returns actual partitioning used, e.g. for restart.
489 //==================================================================
492 DocInfo& doc_info,
493 const bool& report_stats)
494 {
495 // Storage for number of processors and number of elements in global mesh
496 int n_proc = this->communicator_pt()->nproc();
497 int my_rank = this->communicator_pt()->my_rank();
498 int n_element = mesh_pt()->nelement();
499
500 // Vector to be returned
502
503 // Buffer extreme cases
504 if (n_proc == 1) // single-process job - don't do anything
505 {
506 if (report_stats)
507 {
508 std::ostringstream warn_message;
509 warn_message << "WARNING: You've tried to distribute a problem over\n"
510 << "only one processor: this would make METIS crash.\n"
511 << "Ignoring your request for distribution.\n";
513 "Problem::distribute()",
515 }
516 }
517 else if (n_proc > n_element) // more processors than elements
518 {
519 // Throw an error
520 std::ostringstream error_stream;
521 error_stream << "You have tried to distribute a problem\n"
522 << "but there are less elements than processors.\n"
523 << "Please re-run with more elements!\n"
524 << "Please also ensure that actions_before_distribute().\n"
525 << "and actions_after_distribute() are correctly set up.\n"
526 << std::endl;
527 throw OomphLibError(
529 }
530 else
531 {
532 // We only distribute uniformly-refined meshes; buffer the case where
533 // either mesh is not uniformly refined
535 unsigned n_mesh = nsub_mesh();
536 if (n_mesh == 0)
537 {
538 // Check refinement levels
540 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
541 {
542 unsigned min_ref_level = 0;
543 unsigned max_ref_level = 0;
544 mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
545 // If they are not the same
547 {
549 }
550 }
551 }
552 else
553 {
554 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
555 {
556 // Check refinement levels for each mesh individually
557 // (one mesh is allowed to be "more uniformly refined" than another)
560 {
561 unsigned min_ref_level = 0;
562 unsigned max_ref_level = 0;
563 mmesh_pt->get_refinement_levels(min_ref_level, max_ref_level);
564 // If they are not the same
566 {
568 }
569 }
570 }
571 }
572
573 // If any mesh is not uniformly refined
575 {
576 // Again it may make more sense to throw an error here as the user
577 // will probably not be running a problem that is small enough to
578 // fit the whole of on each processor
579 std::ostringstream error_stream;
580 error_stream << "You have tried to distribute a problem\n"
581 << "but at least one of your meshes is no longer\n"
582 << "uniformly refined. In order to preserve the Tree\n"
583 << "and TreeForest structure, Problem::distribute() can\n"
584 << "only be called while meshes are uniformly refined.\n"
585 << std::endl;
586 throw OomphLibError(
588 }
589 else
590 {
591 // Is there any global data? If so, distributing the problem won't work
592 if (nglobal_data() > 0)
593 {
594 std::ostringstream error_stream;
595 error_stream << "You have tried to distribute a problem\n"
596 << "and there is some global data.\n"
597 << "This is not likely to work...\n"
598 << std::endl;
599 throw OomphLibError(error_stream.str(),
602 }
603
604 double t_start = 0;
606 {
608 }
609
610
611#ifdef PARANOID
612 unsigned old_ndof = ndof();
613#endif
614
615 // Need to partition the global mesh before distributing
617
618 // Vector listing the affiliation of each element
619 unsigned nelem = global_mesh_pt->nelement();
621
622 // Number of elements that I'm in charge of, based on any
623 // incoming partitioning
624 unsigned n_my_elements = 0;
625
626 // Have we used the pre-set partitioning
627 bool used_preset_partitioning = false;
628
629 // Partition the mesh, unless the partition has already been passed in
630 // If it hasn't then the sum of all the entries of the vector should be
631 // 0
632 unsigned sum_element_partition = 0;
633 unsigned n_part = element_partition.size();
634 for (unsigned e = 0; e < n_part; e++)
635 {
636 // ... another one for me.
638
640 }
641 if (sum_element_partition == 0)
642 {
643 oomph_info << "INFO: using METIS to partition elements" << std::endl;
646 }
647 else
648 {
649 oomph_info << "INFO: using pre-set partition of elements"
650 << std::endl;
653 }
654
655 // Set the GLOBAL Mesh as being distributed
656 global_mesh_pt->set_communicator_pt(this->communicator_pt());
657
658 double t_end = 0.0;
660 {
662 oomph_info << "Time for partitioning of global mesh: "
663 << t_end - t_start << std::endl;
665 }
666
667 // Store how many elements we had in the various sub-meshes
668 // before actions_before_distribute() (which may empty some of
669 // them).
671 if (n_mesh != 0)
672 {
673 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
674 {
675 unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
677 }
678 }
679
680 // Partitioning complete; call actions before distribute
682
684 {
686 oomph_info << "Time for actions before distribute: "
687 << t_end - t_start << std::endl;
688 }
689
690 // This next bit is cheap -- omit timing
691 // t_start = TimingHelpers::timer();
692
693 // Number of submeshes (NB: some may have been deleted in
694 // actions_after_distribute())
695 n_mesh = nsub_mesh();
696
697
698 // Prepare vector of vectors for submesh element domains
700
701 // The submeshes need to know their own element domains.
702 // Also if any meshes have been emptied we ignore their
703 // partitioning in the vector that we return from here
705 if (n_mesh != 0)
706 {
707 unsigned count = 0;
708 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
709 {
710 unsigned nsub_elem = mesh_pt(i_mesh)->nelement();
713 for (unsigned e = 0; e < nsub_elem_old; e++)
714 {
716 {
719 }
720 // return_element_domain.push_back(element_domain[count]);
721 count++;
722 }
723 }
724 }
725 else
726 {
728 }
729
731 {
733 }
734
735 // Setup the map between "root" element and number in global mesh
736 // (currently used in the load_balance() routines)
737
738 // This map is only established for structured meshes, then we
739 // need to check here the type of mesh
740 if (n_mesh == 0)
741 {
742 // Check if the only one mesh is an structured mesh
743 bool structured_mesh = true;
745 dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
746 if (tri_mesh_pt != 0)
747 {
748 structured_mesh = false;
749 } // if (tri_mesh_pt != 0)
750 if (structured_mesh)
751 {
752 const unsigned n_ele = global_mesh_pt->nelement();
755 for (unsigned e = 0; e < n_ele; e++)
756 {
760 } // for (e<n_ele)
761 } // A TreeBaseMesh mesh
762 } // if (n_mesh==0)
763 else
764 {
765 // If we have submeshes then we only add those elements that
766 // belong to structured meshes, but first compute the number
767 // of total elements in the structured meshes
768 unsigned nglobal_element = 0;
769 // Store which submeshes are structured
770 std::vector<bool> is_structured_mesh(n_mesh);
771 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
772 {
774 dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
775 if (tri_mesh_pt != 0)
776 {
777 // Set the flags to indicate this is not an structured
778 // mesh
779 is_structured_mesh[i_mesh] = false;
780 } // if (tri_mesh_pt != 0)
781 else
782 {
783 // Set the flags to indicate this is an structured
784 // mesh
786 } // else if (tri_mesh_pt != 0)
787 // Check if mesh is an structured mesh
789 {
791 } // A TreeBaseMesh mesh
792 } // for (i_mesh<n_mesh)
793
794 // Once computed the number of elements, then resize the
795 // structure
798 unsigned counter = 0;
799 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
800 {
801 // Check if mesh is an structured mesh
803 {
804 const unsigned n_ele = mesh_pt(i_mesh)->nelement();
805 for (unsigned e = 0; e < n_ele; e++)
806 {
810 // Inrease the global element number
811 counter++;
812 } // for (e<n_ele)
813 } // An structured mesh
814 } // for (i_mesh<n_mesh)
815
816#ifdef PARANOID
818 {
819 std::ostringstream error_stream;
821 << "The number of global elements (" << nglobal_element
822 << ") is not the sameas the number of\nadded elements ("
823 << counter << ") to the Base_mesh_element_pt data "
824 << "structure!!!\n\n";
825 throw OomphLibError(error_stream.str(),
826 "Problem::distribute()",
828 } // if (counter != nglobal_element)
829#endif // #ifdef PARANOID
830
831 } // else if (n_mesh==0)
832
833 // Wipe everything if a pre-determined partitioning
834 // didn't specify ANY elements for this processor
835 // (typically happens during restarts with larger number
836 // of processors -- in this case we really want an empty
837 // processor rather than one with any "kept" halo elements)
840 {
841 oomph_info << "INFO: We're over-ruling the \"keep as halo element\"\n"
842 << " status because the preset partitioning\n"
843 << " didn't place ANY elements on this processor,\n"
844 << " probably because of a restart on a larger \n"
845 << " number of processors\n";
847 }
848
849
850 // Distribute the (sub)meshes (i.e. sort out their halo lookup schemes)
852 if (n_mesh == 0)
853 {
854 global_mesh_pt->distribute(this->communicator_pt(),
855 element_domain,
857 doc_info,
860 }
861 else // There are submeshes, "distribute" each one separately
862 {
863 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
864 {
865 if (report_stats)
866 {
867 oomph_info << "Distributing submesh " << i_mesh << std::endl
868 << "--------------------" << std::endl;
869 }
870 // Set the doc_info number to reflect the submesh
871 doc_info.number() = i_mesh;
873 submesh_element_domain[i_mesh],
875 doc_info,
878 }
879 // Rebuild the global mesh
881 }
882
883 // Null out information associated with deleted elements
884 unsigned n_del = deleted_element_pt.size();
885 for (unsigned e = 0; e < n_del; e++)
886 {
891 }
892
894 {
896 oomph_info << "Time for mesh-level distribution: " << t_end - t_start
897 << std::endl;
899 }
900
901 // Now the problem has been distributed
903
904 // Call actions after distribute
906
908 {
910 oomph_info << "Time for actions after distribute: " << t_end - t_start
911 << std::endl;
913 }
914
915 // Re-assign the equation numbers (incl synchronisation if reqd)
916 unsigned n_dof = assign_eqn_numbers();
917 oomph_info << "Number of equations: " << n_dof << std::endl;
918
920 {
922 oomph_info << "Time for re-assigning eqn numbers (in distribute): "
923 << t_end - t_start << std::endl;
924 }
925
926
927#ifdef PARANOID
928 if (n_dof != old_ndof)
929 {
930 std::ostringstream error_stream;
932 << "Number of dofs in distribute() has changed "
933 << "from " << old_ndof << " to " << n_dof << "\n"
934 << "Check that you've implemented any necessary "
935 "actions_before/after\n"
936 << "distribute functions, e.g. to pin redundant pressure dofs"
937 << " etc.\n";
938 throw OomphLibError(error_stream.str(),
941 }
942#endif
943
944 } // end if to check for uniformly refined mesh(es)
945
946 } // end if to check number of processors vs. number of elements etc.
947
948
949 // Force re-analysis of time spent on assembly each
950 // elemental Jacobian
953
954 // Return the partition vector used in the distribution
956 }
957
958 //==================================================================
959 /// Partition the global mesh, return vector specifying the processor
960 /// number for each element. Virtual so that it can be overloaded by
961 /// any user; the default is to use METIS to perform the partitioning
962 /// (with a bit of cleaning up afterwards to sort out "special cases").
963 //==================================================================
965 DocInfo& doc_info,
967 const bool& report_stats)
968 {
969 // Storage for number of processors and current processor
970 int n_proc = this->communicator_pt()->nproc();
971 int rank = this->communicator_pt()->my_rank();
972
973 std::ostringstream filename;
974 std::ofstream some_file;
975
976 // Doc the original mesh on proc 0
977 //--------------------------------
978 if (doc_info.is_doc_enabled())
979 {
980 if (rank == 0)
981 {
982 filename << doc_info.directory() << "/complete_mesh"
983 << doc_info.number() << ".dat";
984 global_mesh_pt->output(filename.str().c_str(), 5);
985 }
986 }
987
988 // Partition the mesh
989 //-------------------
990 // METIS Objective (0: minimise edge cut; 1: minimise total comm volume)
991 unsigned objective = 0;
992
993 // Do the partitioning
994 unsigned nelem = 0;
995 if (this->communicator_pt()->my_rank() == 0)
996 {
999 }
1001 element_domain.resize(nelem);
1003 nelem,
1005 0,
1006 this->communicator_pt()->mpi_comm());
1007
1008 // On very coarse meshes with larger numbers of processors, METIS
1009 // occasionally returns an element_domain Vector for which a particular
1010 // processor has no elements affiliated to it; the following fixes this
1011
1012 // Convert element_domain to integer storage
1014 for (unsigned e = 0; e < nelem; e++)
1015 {
1017 }
1018
1019 // Global storage for number of elements on each process
1020 int my_number_of_elements = 0;
1022
1023 for (unsigned e = 0; e < nelem; e++)
1024 {
1025 if (int_element_domain[e] == rank)
1026 {
1028 }
1029 }
1030
1031 // Communicate the correct value for each single process into
1032 // the global storage vector
1034 1,
1035 MPI_INT,
1037 1,
1038 MPI_INT,
1039 this->communicator_pt()->mpi_comm());
1040
1041 // If a process has no elements then switch an element with the
1042 // process with the largest number of elements, assuming
1043 // that it still has enough elements left to share
1044 int max_number_of_elements = 0;
1046 for (int d = 0; d < n_proc; d++)
1047 {
1048 if (number_of_elements[d] == 0)
1049 {
1050 // Find the process with maximum number of elements
1051 if (max_number_of_elements <= 1)
1052 {
1053 for (int dd = 0; dd < n_proc; dd++)
1054 {
1056 {
1059 }
1060 }
1061 }
1062
1063 // Check that this number of elements is okay for sharing
1064 if (max_number_of_elements <= 1)
1065 {
1066 // Throw an error if elements can't be shared
1067 std::ostringstream error_stream;
1068 error_stream << "No process has more than 1 element, and\n"
1069 << "at least one process has no elements!\n"
1070 << "Suggest rerunning with more refinement.\n"
1071 << std::endl;
1072 throw OomphLibError(error_stream.str(),
1075 }
1076
1077 // Loop over the element domain vector and switch
1078 // one value for process "process_with_max_elements" with d
1079 for (unsigned e = 0; e < nelem; e++)
1080 {
1082 {
1083 int_element_domain[e] = d;
1084 // Change the numbers associated with these processes
1085 number_of_elements[d]++;
1087 // Reduce the number of elements available on "max" process
1089 // Inform the user that a switch has taken place
1090 if (report_stats)
1091 {
1092 oomph_info << "INFO: Switched element domain at position " << e
1093 << std::endl
1094 << "from process " << process_with_max_elements
1095 << " to process " << d << std::endl
1096 << "which was given no elements by METIS partition"
1097 << std::endl;
1098 }
1099 // Only need to do this once for this element loop, otherwise
1100 // this will take all the elements from "max" process and put them
1101 // in process d, thus leaving essentially the same problem!
1102 break;
1103 }
1104 }
1105 }
1106 }
1107
1108 // Reassign new values to the element_domain vector
1109 for (unsigned e = 0; e < nelem; e++)
1110 {
1112 }
1113
1114 unsigned count_elements = 0;
1115 for (unsigned e = 0; e < nelem; e++)
1116 {
1117 if (int(element_domain[e]) == rank)
1118 {
1120 }
1121 }
1122
1123 if (report_stats)
1124 {
1125 oomph_info << "I have " << count_elements
1126 << " elements from this partition" << std::endl
1127 << std::endl;
1128 }
1129 }
1130
1131 //==================================================================
1132 /// (Irreversibly) prune halo(ed) elements and nodes, usually
1133 /// after another round of refinement, to get rid of
1134 /// excessively wide halo layers. Note that the current
1135 /// mesh will be now regarded as the base mesh and no unrefinement
1136 /// relative to it will be possible once this function
1137 /// has been called.
1138 //==================================================================
1140 const bool& report_stats)
1141 {
1142 // Storage for number of processors and current processor
1143 int n_proc = this->communicator_pt()->nproc();
1144
1145 // Has the problem been distributed yet?
1147 {
1149 << "WARNING: Problem::prune_halo_elements_and_nodes() was called on a "
1150 << "non-distributed Problem!" << std::endl;
1151 oomph_info << "Ignoring your request..." << std::endl;
1152 }
1153 else
1154 {
1155 // There are no halo layers to prune if it's a single-process job
1156 if (n_proc == 1)
1157 {
1159 << "WARNING: You've tried to prune halo layers on a problem\n"
1160 << "with only one processor: this is unnecessary.\n"
1161 << "Ignoring your request." << std::endl
1162 << std::endl;
1163 }
1164 else
1165 {
1166#ifdef PARANOID
1167 unsigned old_ndof = ndof();
1168#endif
1169
1170 double t_start = 0.0;
1172 {
1174 }
1175
1176 // Call actions before distribute
1178
1179 double t_end = 0.0;
1181 {
1183 oomph_info << "Time for actions_before_distribute() in "
1184 << "Problem::prune_halo_elements_and_nodes(): "
1185 << t_end - t_start << std::endl;
1187 }
1188
1189 // Associate all elements with root in current Base mesh
1190 unsigned nel = Base_mesh_element_pt.size();
1191 std::map<GeneralisedElement*, unsigned>
1193 std::vector<bool> old_root_is_halo_or_non_existent(nel, true);
1194 for (unsigned e = 0; e < nel; e++)
1195 {
1196 // Get the base element
1198
1199 // Does it exist locally?
1200 if (base_el_pt != 0)
1201 {
1202 // Check if it's a halo element
1203 if (!base_el_pt->is_halo())
1204 {
1206 }
1207
1208 // Not refineable: It's only the element iself
1210 ref_el_pt = dynamic_cast<RefineableElement*>(base_el_pt);
1211 if (ref_el_pt == 0)
1212 {
1214 }
1215 // Refineable: Get entire tree of elements
1216 else
1217 {
1218 Vector<Tree*> tree_pt;
1219 ref_el_pt->tree_pt()->stick_all_tree_nodes_into_vector(tree_pt);
1220 unsigned ntree = tree_pt.size();
1221 for (unsigned t = 0; t < ntree; t++)
1222 {
1223 old_base_element_number_plus_one[tree_pt[t]->object_pt()] =
1224 e + 1;
1225 }
1226 }
1227 }
1228 }
1229
1230
1232 {
1234 oomph_info << "Time for setup old root elements in "
1235 << "Problem::prune_halo_elements_and_nodes(): "
1236 << t_end - t_start << std::endl;
1238 }
1239
1240
1241 // Now remember the old number of base elements
1242 unsigned nel_base_old = nel;
1243
1244
1245 // Prune the halo elements and nodes of the mesh(es)
1247 unsigned n_mesh = nsub_mesh();
1248 if (n_mesh == 0)
1249 {
1250 // Prune halo elements and nodes for the (single) global mesh
1252 deleted_element_pt, doc_info, report_stats);
1253 }
1254 else
1255 {
1256 // Loop over individual submeshes and prune separately
1257 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
1258 {
1260 deleted_element_pt, doc_info, report_stats);
1261 }
1262
1263 // Rebuild the global mesh
1265 }
1266
1268 {
1270 oomph_info << "Total time for all mesh-level prunes in "
1271 << "Problem::prune_halo_elements_and_nodes(): "
1272 << t_end - t_start << std::endl;
1274 }
1275
1276 // Loop over all elements in newly rebuilt mesh (which contains
1277 // all element in "tree order"), find the roots
1278 // (which are either non-refineable elements or refineable elements
1279 // whose tree representations are TreeRoots)
1280 std::map<FiniteElement*, bool> root_el_done;
1281
1282 // Vector storing vectors of pointers to new base elements associated
1283 // with the same old base element
1286
1287 unsigned n_meshes = n_mesh;
1288 // Change the value for the number of submeshes if there is only
1289 // one mesh so that the loop below works if we have only one
1290 // mesh
1291 if (n_meshes == 0)
1292 {
1293 n_meshes = 1;
1294 }
1295
1296 // Store which submeshes, if there are some are structured
1297 // meshes
1298 std::vector<bool> is_structured_mesh(n_meshes);
1299
1300 // Loop over all elements in the rebuilt mesh, but make sure
1301 // that we are only looping over the structured meshes
1302 nel = 0;
1303 for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1304 {
1306 dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
1307 if (!(tri_mesh_pt != 0))
1308 {
1309 // Mark the mesh as structured mesh
1310 is_structured_mesh[i_mesh] = true;
1311 // Add the number of elements
1312 nel += mesh_pt(i_mesh)->nelement();
1313 } // if (!(tri_mesh_pt!=0))
1314 else
1315 {
1316 // Mark the mesh as nonstructured mesh
1317 is_structured_mesh[i_mesh] = false;
1318 } // else if (!(tri_mesh_pt!=0))
1319 } // for (i_mesh < n_mesh)
1320
1321 // Go for all the meshes (if there are submeshes)
1322 for (unsigned i_mesh = 0; i_mesh < n_meshes; i_mesh++)
1323 {
1324 // Only work with the elements in the mesh if it is an
1325 // structured mesh
1327 {
1328 // Get the number of elements in the submesh
1329 const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
1330 for (unsigned e = 0; e < nele_submesh; e++)
1331 {
1332 // Get the element
1334
1335 // Not refineable: It's definitely a new base element
1337 ref_el_pt = dynamic_cast<RefineableElement*>(el_pt);
1338 if (ref_el_pt == 0)
1339 {
1340 unsigned old_base_el_no =
1344 .push_back(el_pt);
1345 }
1346 // Refineable
1347 else
1348 {
1349 // Is it a tree root (after pruning)? In that case it's
1350 // a new base element
1351 if (dynamic_cast<TreeRoot*>(ref_el_pt->tree_pt()))
1352 {
1353 unsigned old_base_el_no =
1357 .push_back(el_pt);
1358 }
1359 else
1360 {
1361 // Get associated root element
1363 ref_el_pt->tree_pt()->root_pt()->object_pt();
1364
1366 {
1367 root_el_done[root_el_pt] = true;
1368 unsigned old_base_el_no =
1372 .push_back(root_el_pt);
1373 }
1374 }
1375 }
1376 } // for (e < nele_submesh)
1377 } // if (is_structured_mesh[i_mesh])
1378 } // for (i_mesh < n_mesh)
1379
1380 // Create a vector that stores how many new root/base elements
1381 // got spawned from each old root/base element in the global mesh
1383#ifdef PARANOID
1385#endif
1386 for (unsigned e = 0; e < nel_base_old; e++)
1387 {
1390
1391#ifdef PARANOID
1392 // Backup so we can check that halo data was consistent
1394#endif
1395 }
1396
1398 {
1400 oomph_info << "Time for setup of new base elements in "
1401 << "Problem::prune_halo_elements_and_nodes(): "
1402 << t_end - t_start << std::endl;
1404 }
1405
1406 // Now do reduce operation to get information for all
1407 // old root/base elements -- the pruned (halo!) base elements contain
1408 // fewer associated new roots.
1411 &n_new_root[0],
1414 MPI_MAX,
1415 this->communicator_pt()->mpi_comm());
1416
1417
1419 {
1421 oomph_info << "Time for allreduce in "
1422 << "Problem::prune_halo_elements_and_nodes(): "
1423 << t_end - t_start << std::endl;
1425 }
1426
1427 // Find out total number of base elements
1428 unsigned nel_base_new = 0;
1429 for (unsigned e = 0; e < nel_base_old; e++)
1430 {
1431 // Increment
1433
1434#ifdef PARANOID
1435 // If we already had data for this root previously then
1436 // the data ought to be consistent afterwards (since taking
1437 // the max of consistent numbers shouldn't change things -- this
1438 // deals with halo/haloed elements)
1440 {
1441 if (n_new_root_back[e] != 0)
1442 {
1443 if (n_new_root_back[e] != n_new_root[e])
1444 {
1445 std::ostringstream error_stream;
1447 << "Number of new root elements spawned from old root " << e
1448 << ": " << n_new_root[e] << "\nis not consistent"
1449 << " with previous value: " << n_new_root_back[e]
1450 << std::endl;
1451 throw OomphLibError(error_stream.str(),
1454 }
1455 }
1456 }
1457
1458#endif
1459 }
1460
1461 // Reset base_mesh information
1462 Base_mesh_element_pt.clear();
1465
1466 // Now enumerate the new base/root elements consistently
1467 unsigned count = 0;
1468 for (unsigned e = 0; e < nel_base_old; e++)
1469 {
1470 // Old root is non-halo: Just add the new roots into the
1471 // new lookup scheme consecutively
1473 {
1474 // Loop over new root/base element
1475 unsigned n_new_root =
1477 for (unsigned j = 0; j < n_new_root; j++)
1478 {
1479 // Store new root/base element
1484
1485 // Bump counter
1486 count++;
1487 }
1488 }
1489 // Old root element is halo so skip insertion (i.e. leave
1490 // entries in lookup schemes nulled) but increase counter to
1491 // ensure consistency between processors
1492 else
1493 {
1494 unsigned nskip = n_new_root[e];
1495 count += nskip;
1496 }
1497 }
1498
1499 // Re-setup the map between "root" element and number in global mesh
1500 // (used in the load_balance() routines)
1502
1503
1505 {
1507 oomph_info << "Time for finishing off base mesh info "
1508 << "Problem::prune_halo_elements_and_nodes(): "
1509 << t_end - t_start << std::endl;
1511 }
1512
1513
1514 // Call actions after distribute
1516
1517
1519 {
1521 oomph_info << "Time for actions_after_distribute() "
1522 << "Problem::prune_halo_elements_and_nodes(): "
1523 << t_end - t_start << std::endl;
1525 }
1526
1527
1528 // Re-assign the equation numbers (incl synchronisation if reqd)
1529#ifdef PARANOID
1530 unsigned n_dof = assign_eqn_numbers();
1531#else
1533#endif
1534
1535
1537 {
1539 oomph_info << "Time for assign_eqn_numbers() "
1540 << "Problem::prune_halo_elements_and_nodes(): "
1541 << t_end - t_start << std::endl;
1543 }
1544
1545
1546#ifdef PARANOID
1548 {
1549 if (n_dof != old_ndof)
1550 {
1551 std::ostringstream error_stream;
1553 << "Number of dofs in prune_halo_elements_and_nodes() has "
1554 "changed "
1555 << "from " << old_ndof << " to " << n_dof << "\n"
1556 << "Check that you've implemented any necessary "
1557 "actions_before/after"
1558 << "\nadapt/distribute functions, e.g. to pin redundant pressure"
1559 << " dofs etc.\n";
1560 throw OomphLibError(error_stream.str(),
1563 }
1564 }
1565#endif
1566 }
1567 }
1568 }
1569
1570
1571#endif
1572
1573
1574 //===================================================================
1575 /// Build a single (global) mesh from a number
1576 /// of submeshes which are passed as a vector of pointers to the
1577 /// submeshes. The ordering is not necessarily optimal.
1578 //==============================================================
1580 {
1581#ifdef PARANOID
1582 // Has a global mesh already been built
1583 if (Mesh_pt != 0)
1584 {
1585 std::string error_message = "Problem::build_global_mesh() called,\n";
1586 error_message += " but a global mesh has already been built:\n";
1587 error_message += "Problem::Mesh_pt is not zero!\n";
1588
1589 throw OomphLibError(
1591 }
1592 // Check that there are submeshes
1593 if (Sub_mesh_pt.size() == 0)
1594 {
1595 std::string error_message = "Problem::build_global_mesh() called,\n";
1596 error_message += " but there are no submeshes:\n";
1597 error_message += "Problem::Sub_mesh_pt has no entries\n";
1598
1599 throw OomphLibError(
1601 }
1602#endif
1603
1604 // Create an empty mesh
1605 Mesh_pt = new Mesh();
1606
1607 // Call the rebuild function to construct the mesh
1609 }
1610
1611 //====================================================================
1612 /// If one of the submeshes has changed (e.g. by
1613 /// mesh adaptation) we need to update the global mesh.
1614 /// \b Note: The nodes boundary information refers to the
1615 /// boundary numbers within the submesh!
1616 /// N.B. This is essentially the same function as the Mesh constructor
1617 /// that assembles a single global mesh from submeshes
1618 //=====================================================================
1620 {
1621 // Use the function in mesh to merge the submeshes into this one
1623 }
1624
1625
1626 //================================================================
1627 /// Add a timestepper to the problem. The function will automatically
1628 /// create or resize the Time object so that it contains the appropriate
1629 /// number of levels of storage.
1630 //================================================================
1631 void Problem::add_time_stepper_pt(TimeStepper* const& time_stepper_pt)
1632 {
1633 // Add the timestepper to the vector
1635
1636 // Find the number of timesteps required by the timestepper
1637 unsigned ndt = time_stepper_pt->ndt();
1638
1639 // If time has not been allocated, create time object with the
1640 // required number of time steps
1641 if (Time_pt == 0)
1642 {
1643 Time_pt = new Time(ndt);
1644 oomph_info << "Created Time with " << ndt << " timesteps" << std::endl;
1645 }
1646 else
1647 {
1648 // If the required number of time steps is greater than currently stored
1649 // resize the time storage
1650 if (ndt > Time_pt->ndt())
1651 {
1652 Time_pt->resize(ndt);
1653 oomph_info << "Resized Time to include " << ndt << " timesteps"
1654 << std::endl;
1655 }
1656 // Otherwise report that we are OK
1657 else
1658 {
1659 oomph_info << "Time object already has storage for " << ndt
1660 << " timesteps" << std::endl;
1661 }
1662 }
1663
1664 // Pass the pointer to time to the timestepper
1666 }
1667
1668 //================================================================
1669 /// Set the explicit time stepper for the problem and also
1670 /// ensure that a time object has been created.
1671 //================================================================
1673 ExplicitTimeStepper* const& explicit_time_stepper_pt)
1674 {
1675 // Set the explicit time stepper
1677
1678 // If time has not been allocated, create time object with the
1679 // required number of time steps
1680 if (Time_pt == 0)
1681 {
1682 Time_pt = new Time(0);
1683 oomph_info << "Created Time with storage for no previous timestep"
1684 << std::endl;
1685 }
1686 else
1687 {
1688 oomph_info << "Time object already exists " << std::endl;
1689 }
1690 }
1691
1692
1693#ifdef OOMPH_HAS_MPI
1694
1695 //================================================================
1696 /// Set default first and last elements for parallel assembly
1697 /// of non-distributed problem.
1698 //================================================================
1700 {
1702 {
1703 // Minimum number of elements per processor if there are fewer elements
1704 // than processors
1705 unsigned min_el = 10;
1706
1707 // Resize and make default assignments
1708 int n_proc = this->communicator_pt()->nproc();
1709 unsigned n_elements = Mesh_pt->nelement();
1710 First_el_for_assembly.resize(n_proc, 0);
1712
1713 // In the absence of any better knowledge distribute work evenly
1714 // over elements
1715 unsigned range = 0;
1716 unsigned lo_proc = 0;
1717 unsigned hi_proc = n_proc - 1;
1718 if (int(n_elements) >= n_proc)
1719 {
1720 range = unsigned(double(n_elements) / double(n_proc));
1721 }
1722 else
1723 {
1724 range = min_el;
1725 lo_proc = 0;
1726 hi_proc = unsigned(double(n_elements) / double(min_el));
1727 }
1728
1729 for (int p = lo_proc; p <= int(hi_proc); p++)
1730 {
1732
1733 unsigned last_el_plus_one = (p + 1) * range;
1736 }
1737
1738 // Last one needs to incorporate any dangling elements
1739 if (int(n_elements) >= n_proc)
1740 {
1742 }
1743
1744 // Doc
1745 if (n_proc > 1)
1746 {
1748 {
1749 oomph_info << "Problem is not distributed. Parallel assembly of "
1750 << "Jacobian uses default partitioning: " << std::endl;
1751 for (int p = 0; p < n_proc; p++)
1752 {
1754 {
1755 oomph_info << "Proc " << p << " assembles from element "
1756 << First_el_for_assembly[p] << " to "
1757 << Last_el_plus_one_for_assembly[p] - 1 << " \n";
1758 }
1759 else
1760 {
1761 oomph_info << "Proc " << p << " assembles no elements\n";
1762 }
1763 }
1764 }
1765 }
1766 }
1767 }
1768
1769
1770 //=======================================================================
1771 /// Helper function to re-assign the first and last elements to be
1772 /// assembled by each processor during parallel assembly for
1773 /// non-distributed problem.
1774 //=======================================================================
1776 {
1777 // Wait until all processes have completed/timed their assembly
1779
1780 // Storage for number of processors and current processor
1781 int n_proc = this->communicator_pt()->nproc();
1782 int rank = this->communicator_pt()->my_rank();
1783
1784 // Don't bother to update if we've got fewer elements than
1785 // processors
1786 unsigned nel = Elemental_assembly_time.size();
1787 if (int(nel) < n_proc)
1788 {
1789 oomph_info << "Not re-computing distribution of elemental assembly\n"
1790 << "because there are fewer elements than processors\n";
1791 return;
1792 }
1793
1794 // Setup vectors storing the number of element timings to be sent
1795 // and the offset in the final vector
1798 int offset = 0;
1799 for (int p = 0; p < n_proc; p++)
1800 {
1801 // Default distribution of labour
1802 unsigned el_lo = First_el_for_assembly[p];
1803 unsigned el_hi = Last_el_plus_one_for_assembly[p] - 1;
1804
1805 // Number of timings to be sent and offset from start in
1806 // final vector
1807 receive_count[p] = el_hi - el_lo + 1;
1808 displacement[p] = offset;
1809 offset += el_hi - el_lo + 1;
1810 }
1811
1812 // Make temporary c-style array to avoid over-writing in Gatherv below
1813 double* el_ass_time = new double[nel];
1814 for (unsigned e = 0; e < nel; e++)
1815 {
1817 }
1818
1819 // Gather timings on root processor
1820 unsigned nel_local =
1823 nel_local,
1824 MPI_DOUBLE,
1826 &receive_count[0],
1827 &displacement[0],
1828 MPI_DOUBLE,
1829 0,
1830 this->communicator_pt()->mpi_comm());
1831 delete[] el_ass_time;
1832
1833 // Vector of first and last elements for each processor
1835 for (int p = 0; p < n_proc; p++)
1836 {
1837 first_and_last_element[p].resize(2);
1838 }
1839
1840 // Re-distribute work
1841 if (rank == 0)
1842 {
1844 {
1846 << std::endl
1847 << "Re-assigning distribution of element assembly over processors:"
1848 << std::endl;
1849 }
1850
1851 // Get total assembly time
1852 double total = 0.0;
1853 unsigned n_elements = Mesh_pt->nelement();
1854 for (unsigned e = 0; e < n_elements; e++)
1855 {
1857 }
1858
1859 // Target load per processor
1860 double target_load = total / double(n_proc);
1861
1862 // We're on the root processor: Always start with the first element
1863 int proc = 0;
1864 first_and_last_element[0][0] = 0;
1865
1866 // Highest element we can help ourselves to if we want to leave
1867 // at least one element for all subsequent processors
1868 unsigned max_el_avail = n_elements - n_proc;
1869
1870 // Initialise total work allocated
1871 total = 0.0;
1872 for (unsigned e = 0; e < n_elements; e++)
1873 {
1875
1876 // Once we have reached the target load or we've used up practically
1877 // all the elements...
1878 if ((total > target_load) || (e == max_el_avail))
1879 {
1880 // Last element for current processor
1882
1883 // Provided that we are not on the last processor
1884 if (proc < (n_proc - 1))
1885 {
1886 // Set first element for next one
1887 first_and_last_element[proc + 1][0] = e + 1;
1888
1889 // Move on to the next processor
1890 proc++;
1891 }
1892
1893 // Can have one more...
1894 max_el_avail++;
1895
1896 // Re-initialise the time
1897 total = 0.0;
1898 } // end of test for "total exceeds target"
1899 }
1900
1901
1902 // Last element for last processor
1904
1905
1906 // The following block should probably be paranoidified away
1907 // but we've screwed the logic up so many times that I feel
1908 // it's safer to keep it...
1909 bool wrong = false;
1910 std::ostringstream error_stream;
1911 for (int p = 0; p < n_proc - 1; p++)
1912 {
1916 {
1917 wrong = true;
1918 error_stream << "Error: First/last element of proc " << p << ": "
1920 << std::endl;
1921 }
1922 unsigned first_of_next = first_and_last_element[p + 1][0];
1923 if (first_of_next != (last_of_current + 1))
1924 {
1925 wrong = true;
1926 error_stream << "Error: First element of proc " << p + 1 << ": "
1927 << first_of_next << " and last element of proc " << p
1928 << ": " << last_of_current << std::endl;
1929 }
1930 }
1931 if (wrong)
1932 {
1933 throw OomphLibError(
1935 }
1936
1937
1938 // THIS TIDY UP SHOULD NO LONGER BE REQUIRED AND CAN GO AT SOME POINT
1939
1940 // //If we haven't got to the end of the processor list then
1941 // //need to shift things about slightly because the processors
1942 // //at the end will be empty.
1943 // //This can occur when you have very fast assembly times and the
1944 // //rounding errors mean that the targets are achieved before all
1945 // processors
1946 // //have been visited.
1947 // //Happens a lot when you massively oversubscribe the CPUs (which was
1948 // //only ever for testing!)
1949 // if (proc!=n_proc-1)
1950 // {
1951 // oomph_info
1952 // << "First pass did not allocate elements on every processor\n";
1953 // oomph_info <<
1954 // "Moving elements so that each processor has at least one\n";
1955
1956 // //Work out number of empty processos
1957 // unsigned n_empty_processors = n_proc - proc + 1;
1958
1959 // //Loop over the processors that do have elements
1960 // //and work out how many we need to steal elements from
1961 // unsigned n_element_on_processors=0;
1962 // do
1963 // {
1964 // //Step down the processors
1965 // --proc;
1966 // //Add the current processor to the number of empty processors
1967 // //because the elements have to be shared between processors
1968 // //including the one(s) on which they are currently stored.
1969 // ++n_empty_processors;
1970 // n_element_on_processors +=
1971 // (first_and_last_element[proc][1] -
1972 // first_and_last_element[proc][0] + 1);
1973 // }
1974 // while(n_element_on_processors < n_empty_processors);
1975
1976 // //Should now be able to put one element on each processor
1977 // //Start from the end and do so
1978 // unsigned current_element = n_elements-1;
1979 // for(int p=n_proc-1;p>proc;p--)
1980 // {
1981 // first_and_last_element[p][1] = current_element;
1982 // first_and_last_element[p][0] = --current_element;
1983 // }
1984
1985 // //Now for the last processor we touched, just adjust the final value
1986 // first_and_last_element[proc][1] = current_element;
1987 // }
1988 // //Otherwise just put the rest of the elements on the final
1989 // //processor
1990 // else
1991 // {
1992 // // Last one
1993 // first_and_last_element[n_proc-1][1]=n_elements-1;
1994 // }
1995
1996
1997 // END PRESUMED-TO-BE-UNNECESSARY BLOCK...
1998
1999
2000 // Now communicate the information
2001
2002 // Set local informationt for this (root) processor
2005
2007 {
2008 oomph_info << "Processor " << 0 << " assembles Jacobians"
2009 << " from elements " << first_and_last_element[0][0]
2010 << " to " << first_and_last_element[0][1] << " "
2011 << std::endl;
2012 }
2013
2014 // Only now can we send the information to the other processors
2015 for (int p = 1; p < n_proc; ++p)
2016 {
2018 2,
2019 MPI_INT,
2020 p,
2021 0,
2022 this->communicator_pt()->mpi_comm());
2023
2024
2026 {
2027 oomph_info << "Processor " << p << " assembles Jacobians"
2028 << " from elements " << first_and_last_element[p][0]
2029 << " to " << first_and_last_element[p][1] << " "
2030 << std::endl;
2031 }
2032 }
2033 }
2034 // Receive first and last element from root on non-master processors
2035 else
2036 {
2037 Vector<int> aux(2);
2039 MPI_Recv(&aux[0],
2040 2,
2041 MPI_INT,
2042 0,
2043 0,
2044 this->communicator_pt()->mpi_comm(),
2045 &status);
2048 }
2049
2050 // Wipe all others
2051 for (int p = 0; p < n_proc; p++)
2052 {
2053 if (p != rank)
2054 {
2057 }
2058 }
2059
2060 // The equations assembled by this processor may have changed so
2061 // we must resize the sparse assemble with arrays previous allocation
2063 }
2064
2065#endif
2066
2067 //================================================================
2068 /// Assign all equation numbers for problem: Deals with global
2069 /// data (= data that isn't attached to any elements) and then
2070 /// does the equation numbering for the elements. Bool argument
2071 /// can be set to false to ignore assigning local equation numbers
2072 /// (necessary in the parallel implementation of locate_zeta
2073 /// between multiple meshes).
2074 //================================================================
2076 const bool& assign_local_eqn_numbers)
2077 {
2078 // Check that the global mesh has been build
2079#ifdef PARANOID
2080 if (Mesh_pt == 0)
2081 {
2082 std::ostringstream error_stream;
2083 error_stream << "Global mesh does not exist, so equation numbers cannot "
2084 "be assigned.\n";
2085 // Check for sub meshes
2086 if (nsub_mesh() == 0)
2087 {
2088 error_stream << "There aren't even any sub-meshes in the Problem.\n"
2089 << "You can set the global mesh directly by using\n"
2090 << "Problem::mesh_pt() = my_mesh_pt;\n"
2091 << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2092 << "to add a sub mesh.\n";
2093 }
2094 else
2095 {
2096 error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2097 }
2098 error_stream << "You need to call Problem::build_global_mesh() to create "
2099 "a global mesh\n"
2100 << "from the sub-meshes.\n\n";
2101
2102 throw OomphLibError(
2104 }
2105#endif
2106
2107 // Number of submeshes
2108 unsigned n_sub_mesh = Sub_mesh_pt.size();
2109
2110#ifdef OOMPH_HAS_MPI
2111
2112 // Storage for number of processors
2113 int n_proc = this->communicator_pt()->nproc();
2114
2115
2116 if (n_proc > 1)
2117 {
2118 // Force re-analysis of time spent on assembly each
2119 // elemental Jacobian
2122 }
2123 else
2124 {
2126 }
2127
2128 // Re-distribution of elements over processors during assembly
2129 // must be recomputed
2131 {
2132 // Set default first and last elements for parallel assembly
2133 // of non-distributed problem.
2135 }
2136
2137#endif
2138
2139
2140 double t_start = 0.0;
2142 {
2144 }
2145
2146 // Loop over all elements in the mesh and set up any additional
2147 // dependencies that they may have (e.g. storing the geometric
2148 // Data, i.e. Data that affects an element's shape in elements
2149 // with algebraic node-update functions
2150 unsigned nel = Mesh_pt->nelement();
2151 for (unsigned e = 0; e < nel; e++)
2152 {
2154 }
2155
2156#ifdef OOMPH_HAS_MPI
2157 // Complete setup of dependencies for external halo elements too
2158 unsigned n_mesh = this->nsub_mesh();
2159 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
2160 {
2161 for (int iproc = 0; iproc < n_proc; iproc++)
2162 {
2164 for (unsigned e = 0; e < n_ext_halo_el; e++)
2165 {
2169 }
2170 }
2171 }
2172#endif
2173
2174
2175 double t_end = 0.0;
2177 {
2180 << "Time for complete setup of dependencies in assign_eqn_numbers: "
2181 << t_end - t_start << std::endl;
2182 }
2183
2184
2185 // Initialise number of dofs for reserve below
2186 unsigned n_dof = 0;
2187
2188 // Potentially loop over remainder of routine, possible re-visiting all
2189 // those parts that must be redone, following the removal of duplicate
2190 // external halo data.
2191 for (unsigned loop_count = 0; loop_count < 2; loop_count++)
2192 {
2193 //(Re)-set the dof pointer to zero length because entries are
2194 // pushed back onto it -- if it's not reset here then we get into
2195 // trouble during mesh refinement when we reassign all dofs
2196 Dof_pt.resize(0);
2197
2198 // Reserve from previous allocation if we're going around again
2199 Dof_pt.reserve(n_dof);
2200
2201 // Reset the equation number
2202 unsigned long equation_number = 0;
2203
2204 // Now set equation numbers for the global Data
2205 unsigned Nglobal_data = nglobal_data();
2206 for (unsigned i = 0; i < Nglobal_data; i++)
2207 {
2208 Global_data_pt[i]->assign_eqn_numbers(equation_number, Dof_pt);
2209 }
2210
2212 {
2214 }
2215
2216 // Call assign equation numbers on the global mesh
2218
2219 // Deal with the spine meshes additional numbering
2220 // If there is only one mesh
2221 if (n_sub_mesh == 0)
2222 {
2223 if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2224 {
2225 n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2226 }
2227 }
2228 // Otherwise loop over the sub meshes
2229 else
2230 {
2231 // Assign global equation numbers first
2232 for (unsigned i = 0; i < n_sub_mesh; i++)
2233 {
2234 if (SpineMesh* const spine_mesh_pt =
2235 dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2236 {
2237 n_dof = spine_mesh_pt->assign_global_spine_eqn_numbers(Dof_pt);
2238 }
2239 }
2240 }
2241
2243 {
2246 << "Time for assign_global_eqn_numbers in assign_eqn_numbers: "
2247 << t_end - t_start << std::endl;
2249 }
2250
2251
2252#ifdef OOMPH_HAS_MPI
2253
2254 // reset previous allocation
2256
2257 // Only synchronise if the problem has actually been
2258 // distributed.
2260 {
2261 // Synchronise the equation numbers and return the total
2262 // number of degrees of freedom in the overall problem
2263 // Do not assign local equation numbers -- we're doing this
2264 // below.
2266 }
2267 // ..else just setup the Dof_distribution_pt
2268 // NOTE: this is setup by synchronise_eqn_numbers(...)
2269 // if Problem_has_been_distributed
2270 else
2271#endif
2272 {
2274 }
2275
2277 {
2279 oomph_info << "Time for Problem::synchronise_eqn_numbers in "
2280 << "Problem::assign_eqn_numbers: " << t_end - t_start
2281 << std::endl;
2282 }
2283
2284
2285#ifdef OOMPH_HAS_MPI
2286
2287
2288 // Now remove duplicate data in external halo elements
2290 {
2292 {
2294 }
2295
2296 // Monitor if we've actually changed anything
2297 bool actually_removed_some_data = false;
2298
2299 // Only do it once!
2300 if (loop_count == 0)
2301 {
2302 if (n_sub_mesh == 0)
2303 {
2305 }
2306 else
2307 {
2308 for (unsigned i = 0; i < n_sub_mesh; i++)
2309 {
2310 bool tmp_actually_removed_some_data = false;
2315 }
2316 }
2317 }
2318
2319
2321 {
2323 std::stringstream tmp;
2324 tmp << "Time for calls to Problem::remove_duplicate_data in "
2325 << "Problem::assign_eqn_numbers: " << t_end - t_start
2326 << " ; have ";
2328 {
2329 tmp << " not ";
2330 }
2331 tmp << " removed some/any data.\n";
2332 oomph_info << tmp.str();
2334 }
2335
2336 // Break out of the loop if we haven't done anything here.
2337 unsigned status = 0;
2339
2340 // Allreduce to check if anyone has removed any data
2341 unsigned overall_status = 0;
2344 1,
2346 MPI_MAX,
2347 this->communicator_pt()->mpi_comm());
2348
2349
2351 {
2353 std::stringstream tmp;
2354 tmp
2355 << "Time for MPI_Allreduce after Problem::remove_duplicate_data in "
2356 << "Problem::assign_eqn_numbers: " << t_end - t_start << std::endl;
2357 oomph_info << tmp.str();
2359 }
2360
2361 // Bail out if we haven't done anything here
2362 if (overall_status != 1)
2363 {
2364 break;
2365 }
2366
2367 // Big tidy up: Remove null pointers from halo/haloed node storage
2368 // for all meshes (this involves comms and therefore must be
2369 // performed outside loop over meshes so the all-to-all is only
2370 // done once)
2372
2373 // Time it...
2375 {
2376 double t_end = TimingHelpers::timer();
2377 oomph_info << "Total time for "
2378 << "Problem::remove_null_pointers_from_external_halo_node_"
2379 "storage(): "
2380 << t_end - t_start << std::endl;
2381 }
2382 }
2383 else
2384 {
2385 // Problem not distributed; no need for another loop
2386 break;
2387 }
2388
2389#else
2390
2391 // Serial run: Again no need for a second loop
2392 break;
2393
2394#endif
2395
2396 } // end of loop over fcts that need to be re-executed if
2397 // we've removed duplicate external data
2398
2399
2400 // Resize the sparse assemble with arrays previous allocation
2402
2403
2405 {
2407 }
2408
2409 // Finally assign local equations
2410 if (assign_local_eqn_numbers)
2411 {
2412 if (n_sub_mesh == 0)
2413 {
2415 }
2416 else
2417 {
2418 for (unsigned i = 0; i < n_sub_mesh; i++)
2419 {
2422 }
2423 }
2424 }
2425
2427 {
2429 oomph_info << "Total time for all Mesh::assign_local_eqn_numbers in "
2430 << "Problem::assign_eqn_numbers: " << t_end - t_start
2431 << std::endl;
2432 }
2433
2434
2435 // and return the total number of DOFs
2436 return n_dof;
2437 }
2438 //================================================================
2439 /// Function to describe the dofs in terms of the global
2440 /// equation number, i.e. what type of value (nodal value of
2441 /// a Node; value in a Data object; value of internal Data in an
2442 /// element; etc) is the unknown with a certain global equation number.
2443 /// Output stream defaults to oomph_info.
2444 //================================================================
2445 void Problem::describe_dofs(std::ostream& out) const
2446 {
2447 // Check that the global mesh has been build
2448#ifdef PARANOID
2449 if (Mesh_pt == 0)
2450 {
2451 std::ostringstream error_stream;
2453 << "Global mesh does not exist, so equation numbers cannot be found.\n";
2454 // Check for sub meshes
2455 if (nsub_mesh() == 0)
2456 {
2457 error_stream << "There aren't even any sub-meshes in the Problem.\n"
2458 << "You can set the global mesh directly by using\n"
2459 << "Problem::mesh_pt() = my_mesh_pt;\n"
2460 << "OR you can use Problem::add_sub_mesh(mesh_pt); "
2461 << "to add a sub mesh.\n";
2462 }
2463 else
2464 {
2465 error_stream << "There are " << nsub_mesh() << " sub-meshes.\n";
2466 }
2467 error_stream << "You need to call Problem::build_global_mesh() to create "
2468 "a global mesh\n"
2469 << "from the sub-meshes.\n\n";
2470
2471 throw OomphLibError(
2473 }
2474#endif
2475
2476 out
2477 << "Although this program will describe the degrees of freedom in the \n"
2478 << "problem, it will do so using the typedef for the elements. This is \n"
2479 << "not neccesarily human readable, but there is a solution.\n"
2480 << "Pipe your program's output through c++filt, with the argument -t.\n"
2481 << "e.g. \"./two_d_multi_poisson | c++filt -t > ReadableOutput.txt\".\n "
2482 << "(Disregarding the quotes)\n\n\n";
2483
2484 out << "Classifying Global Equation Numbers" << std::endl;
2485 out << std::string(80, '-') << std::endl;
2486
2487 // Number of submeshes
2488 unsigned n_sub_mesh = Sub_mesh_pt.size();
2489
2490 // Classify Global dofs
2491 unsigned Nglobal_data = nglobal_data();
2492 for (unsigned i = 0; i < Nglobal_data; i++)
2493 {
2494 std::stringstream conversion;
2495 conversion << " in Global Data " << i << ".";
2496 std::string in(conversion.str());
2498 }
2499
2500 // Put string in limiting scope.
2501 {
2502 // Descend into assignment for mesh.
2503 std::string in(" in Problem's Only Mesh.");
2505 }
2506
2507 // Deal with the spine meshes additional numbering:
2508 // If there is only one mesh:
2509 if (n_sub_mesh == 0)
2510 {
2511 if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
2512 {
2513 std::string in(" in Problem's Only SpineMesh.");
2514 spine_mesh_pt->describe_spine_dofs(out, in);
2515 }
2516 }
2517 // Otherwise loop over the sub meshes
2518 else
2519 {
2520 // Assign global equation numbers first
2521 for (unsigned i = 0; i < n_sub_mesh; i++)
2522 {
2523 if (SpineMesh* const spine_mesh_pt =
2524 dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
2525 {
2526 std::stringstream conversion;
2527 conversion << " in Sub-SpineMesh " << i << ".";
2528 std::string in(conversion.str());
2529 spine_mesh_pt->describe_spine_dofs(out, in);
2530 } // end if.
2531 } // end for.
2532 } // end else.
2533
2534
2535 out << std::string(80, '\\') << std::endl;
2536 out << std::string(80, '\\') << std::endl;
2537 out << std::string(80, '\\') << std::endl;
2538 out << "Classifying global eqn numbers in terms of elements." << std::endl;
2539 out << std::string(80, '-') << std::endl;
2540 out << "Eqns | Source" << std::endl;
2541 out << std::string(80, '-') << std::endl;
2542
2543 if (n_sub_mesh == 0)
2544 {
2545 std::string in(" in Problem's Only Mesh.");
2547 }
2548 else
2549 {
2550 for (unsigned i = 0; i < n_sub_mesh; i++)
2551 {
2552 std::stringstream conversion;
2553 conversion << " in Sub-Mesh " << i << ".";
2554 std::string in(conversion.str());
2556 } // End for
2557 } // End else
2558 } // End problem::describe_dofs(...)
2559
2560
2561 //================================================================
2562 /// Get the vector of dofs, i.e. a vector containing the current
2563 /// values of all unknowns.
2564 //================================================================
2566 {
2567 // Find number of dofs
2568 const unsigned long n_dof = ndof();
2569
2570 // Resize the vector
2571 dofs.build(Dof_distribution_pt, 0.0);
2572
2573 // Copy dofs into vector
2574 for (unsigned long l = 0; l < n_dof; l++)
2575 {
2576 dofs[l] = *Dof_pt[l];
2577 }
2578 }
2579
2580 /// Get history values of dofs in a double vector.
2581 void Problem::get_dofs(const unsigned& t, DoubleVector& dofs) const
2582 {
2583#ifdef PARANOID
2584 if (distributed())
2585 {
2586 throw OomphLibError("Not designed for distributed problems",
2589 // might work, not sure
2590 }
2591#endif
2592
2593 // Resize the vector
2594 dofs.build(Dof_distribution_pt, 0.0);
2595
2596 // First deal with global data
2597 unsigned Nglobal_data = nglobal_data();
2598 for (unsigned i = 0; i < Nglobal_data; i++)
2599 {
2600 for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
2601 {
2602 // For each data get the equation number and copy out the value.
2603 int eqn_number = Global_data_pt[i]->eqn_number(j);
2604 if (eqn_number >= 0)
2605 {
2606 dofs[eqn_number] = Global_data_pt[i]->value(t, j);
2607 }
2608 }
2609 }
2610
2611 // Next element internal data
2612 for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
2613 {
2615 for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
2616 {
2617 Data* d_pt = ele_pt->internal_data_pt(j);
2618 for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
2619 {
2620 int eqn_number = d_pt->eqn_number(k);
2621 if (eqn_number >= 0)
2622 {
2623 dofs[eqn_number] = d_pt->value(t, k);
2624 }
2625 }
2626 }
2627 }
2628
2629 // Now the nodes
2630 for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
2631 {
2632 Node* node_pt = mesh_pt()->node_pt(i);
2633 for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
2634 {
2635 // For each node get the equation number and copy out the value.
2636 int eqn_number = node_pt->eqn_number(j);
2637 if (eqn_number >= 0)
2638 {
2639 dofs[eqn_number] = node_pt->value(t, j);
2640 }
2641 }
2642 }
2643 }
2644
2645
2646#ifdef OOMPH_HAS_MPI
2647
2648 //=======================================================================
2649 /// Private helper function to remove repeated data
2650 /// in external haloed elements associated with specified mesh.
2651 /// Bool is true if some data was removed -- this usually requires
2652 /// re-running through certain parts of the equation numbering procedure.
2653 //======================================================================
2656 {
2657 // // // Taken out again by MH -- clutters up output
2658 // // Doc timings if required
2659 // double t_start=0.0;
2660 // if (Global_timings::Doc_comprehensive_timings)
2661 // {
2662 // t_start=TimingHelpers::timer();
2663 // }
2664
2665 int n_proc = this->communicator_pt()->nproc();
2666 int my_rank = this->communicator_pt()->my_rank();
2667
2668 // Initialise
2669 actually_removed_some_data = false;
2670
2671 // Each individual container of external halo nodes has unique
2672 // nodes/equation numbers, but there may be some duplication between
2673 // two or more different containers; the following code checks for this
2674 // and removes the duplication by overwriting any data point with an already
2675 // existing eqn number with the original data point which had the eqn no.
2676
2677 // // Storage for existing nodes, enumerated by first non-negative
2678 // // global equation number
2679 // unsigned n_dof=ndof();
2680
2681 // Note: This used to be
2682 // Vector<Node*> global_node_pt(n_dof,0);
2683 // but this is a total killer! Memory allocation is extremely
2684 // costly and only relatively few entries are used so use
2685 // map:
2686 std::map<unsigned, Node*> global_node_pt;
2687
2688 // Only do each retained node once
2689 std::map<Node*, bool> node_done;
2690
2691 // Loop over existing "normal" elements in mesh
2692 unsigned n_element = mesh_pt->nelement();
2693 for (unsigned e = 0; e < n_element; e++)
2694 {
2696 dynamic_cast<FiniteElement*>(mesh_pt->element_pt(e));
2697 if (el_pt != 0)
2698 {
2699 // Loop over nodes
2700 unsigned n_node = el_pt->nnode();
2701 for (unsigned j = 0; j < n_node; j++)
2702 {
2703 Node* nod_pt = el_pt->node_pt(j);
2704
2705 // Have we already done the node?
2706 if (!node_done[nod_pt])
2707 {
2708 node_done[nod_pt] = true;
2709
2710 // Loop over values stored at node (if any) to find
2711 // the first non-negative eqn number
2713 unsigned n_val = nod_pt->nvalue();
2714 for (unsigned i_val = 0; i_val < n_val; i_val++)
2715 {
2716 int eqn_no = nod_pt->eqn_number(i_val);
2717 if (eqn_no >= 0)
2718 {
2720 break;
2721 }
2722 }
2723
2724 // If we haven't found a non-negative eqn number check
2725 // eqn numbers associated with solid data (if any)
2727 {
2728 // Is it a solid node?
2729 SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2730 if (solid_nod_pt != 0)
2731 {
2732 // Loop over values stored at node (if any) to find
2733 // the first non-negative eqn number
2734 unsigned n_val = solid_nod_pt->variable_position_pt()->nvalue();
2735 for (unsigned i_val = 0; i_val < n_val; i_val++)
2736 {
2737 int eqn_no =
2738 solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2739 if (eqn_no >= 0)
2740 {
2742 break;
2743 }
2744 }
2745 }
2746 }
2747
2748 // Associate node with first non negative global eqn number
2750 {
2752 nod_pt;
2753 }
2754
2755
2756 // Take into account master nodes too
2757 if (dynamic_cast<RefineableElement*>(el_pt) != 0)
2758 {
2759 int n_cont_int_values = dynamic_cast<RefineableElement*>(el_pt)
2760 ->ncont_interpolated_values();
2761 for (int i_cont = -1; i_cont < n_cont_int_values; i_cont++)
2762 {
2763 if (nod_pt->is_hanging(i_cont))
2764 {
2765 HangInfo* hang_pt = nod_pt->hanging_pt(i_cont);
2766 unsigned n_master = hang_pt->nmaster();
2767 for (unsigned m = 0; m < n_master; m++)
2768 {
2769 Node* master_nod_pt = hang_pt->master_node_pt(m);
2771 {
2772 node_done[master_nod_pt] = true;
2773
2774 // Loop over values stored at node (if any) to find
2775 // the first non-negative eqn number
2777 unsigned n_val = master_nod_pt->nvalue();
2778 for (unsigned i_val = 0; i_val < n_val; i_val++)
2779 {
2781 if (eqn_no >= 0)
2782 {
2784 break;
2785 }
2786 }
2787
2788 // If we haven't found a non-negative eqn number check
2789 // eqn numbers associated with solid data (if any)
2791 {
2792 // If this master is a SolidNode then add its extra
2793 // eqn numbers
2795 dynamic_cast<SolidNode*>(master_nod_pt);
2796 if (master_solid_nod_pt != 0)
2797 {
2798 // Loop over values stored at node (if any) to find
2799 // the first non-negative eqn number
2800 unsigned n_val =
2801 master_solid_nod_pt->variable_position_pt()
2802 ->nvalue();
2803 for (unsigned i_val = 0; i_val < n_val; i_val++)
2804 {
2805 int eqn_no =
2806 master_solid_nod_pt->variable_position_pt()
2807 ->eqn_number(i_val);
2808 if (eqn_no >= 0)
2809 {
2811 eqn_no + 1;
2812 break;
2813 }
2814 }
2815 }
2816 }
2817 // Associate node with first non negative global
2818 // eqn number
2820 {
2822 1] = master_nod_pt;
2823 }
2824
2825 } // End of not-yet-done hang node
2826 }
2827 }
2828 }
2829 }
2830 } // endif for node already done
2831 } // End of loop over nodes
2832 } // End of FiniteElement
2833
2834 // Internal data equation numbers do not need to be added since
2835 // internal data cannot be shared between distinct elements, so
2836 // internal data on locally-stored elements can never be halo.
2837 }
2838
2839 // Set to record duplicate nodes scheduled to be killed
2840 std::set<Node*> killed_nodes;
2841
2842 // Now loop over the other processors from highest to lowest
2843 // (i.e. if there is a duplicate between these containers
2844 // then this will use the node on the highest numbered processor)
2845 for (int iproc = n_proc - 1; iproc >= 0; iproc--)
2846 {
2847 // Don't have external halo elements with yourself!
2848 if (iproc != my_rank)
2849 {
2850 // Loop over external halo elements with iproc
2851 // to remove the duplicates
2853 for (unsigned e_ext = 0; e_ext < n_element; e_ext++)
2854 {
2857 if (finite_ext_el_pt != 0)
2858 {
2859 // Loop over nodes
2860 unsigned n_node = finite_ext_el_pt->nnode();
2861 for (unsigned j = 0; j < n_node; j++)
2862 {
2864
2865 // Loop over values stored at node (if any) to find
2866 // the first non-negative eqn number
2868 unsigned n_val = nod_pt->nvalue();
2869 for (unsigned i_val = 0; i_val < n_val; i_val++)
2870 {
2871 int eqn_no = nod_pt->eqn_number(i_val);
2872 if (eqn_no >= 0)
2873 {
2875 break;
2876 }
2877 }
2878
2879 // If we haven't found a non-negative eqn number check
2880 // eqn numbers associated with solid data (if any)
2882 {
2883 // Is it a solid node?
2884 SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
2885 if (solid_nod_pt != 0)
2886 {
2887 // Loop over values stored at node (if any) to find
2888 // the first non-negative eqn number
2889 unsigned n_val =
2890 solid_nod_pt->variable_position_pt()->nvalue();
2891 for (unsigned i_val = 0; i_val < n_val; i_val++)
2892 {
2893 int eqn_no =
2894 solid_nod_pt->variable_position_pt()->eqn_number(i_val);
2895 if (eqn_no >= 0)
2896 {
2898 break;
2899 }
2900 }
2901 }
2902 }
2903
2904 // Identified which node we're dealing with via first non-negative
2905 // global eqn number (if there is none, everything is pinned
2906 // and we don't give a damn...)
2908 {
2911
2912 // Does this node already exist?
2913 if (existing_node_pt != 0)
2914 {
2915 // Record that we're about to cull one
2917
2918 // It's a duplicate, so store the duplicated one for
2919 // later killing...
2922 {
2923 // Remove node from all boundaries
2924 std::set<unsigned>* boundaries_pt;
2925 duplicated_node_pt->get_boundaries_pt(boundaries_pt);
2926 if (boundaries_pt != 0)
2927 {
2929 unsigned nb = (*boundaries_pt).size();
2930 bound.reserve(nb);
2931 for (std::set<unsigned>::iterator it =
2932 (*boundaries_pt).begin();
2933 it != (*boundaries_pt).end();
2934 it++)
2935 {
2936 bound.push_back((*it));
2937 }
2938 for (unsigned i = 0; i < nb; i++)
2939 {
2942 }
2943 }
2944
2945 // Get ready to kill it
2947 unsigned i_proc = unsigned(iproc);
2950 }
2951
2952
2953 // Note: For now we're leaving the "dangling" (no longer
2954 // accessed masters where they are; they get cleaned
2955 // up next time we delete all the external storage
2956 // for the meshes so it's a temporary "leak" only...
2957 // At some point we should probably delete them properly too
2958
2959#ifdef PARANOID
2960
2961 // Check that hang status of exiting and replacement node
2962 // matches
2963 if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
2964 {
2966 dynamic_cast<RefineableElement*>(finite_ext_el_pt)
2967 ->ncont_interpolated_values();
2968 for (int i_cont = -1; i_cont < n_cont_inter_values;
2969 i_cont++)
2970 {
2971 unsigned n_master_orig = 0;
2973 {
2976 ->nmaster();
2977
2978 // Temporary leak: Resolve like this:
2979 // loop over all external halo nodes and identify the
2980 // the ones that are still reached by any of the
2981 // external elements. Kill the dangling ones.
2982 }
2983 unsigned n_master_replace = 0;
2984 if (existing_node_pt->is_hanging(i_cont))
2985 {
2987 existing_node_pt->hanging_pt(i_cont)->nmaster();
2988 }
2989
2991 {
2992 std::ostringstream error_stream;
2994 << "Number of master nodes for node to be replaced, "
2995 << n_master_orig << ", doesn't match"
2996 << "those of replacement node, " << n_master_replace
2997 << " for i_cont=" << i_cont << std::endl;
2998 {
3000 << "Nodal coordinates of replacement node:";
3001 unsigned ndim = existing_node_pt->ndim();
3002 for (unsigned i = 0; i < ndim; i++)
3003 {
3004 error_stream << existing_node_pt->x(i) << " ";
3005 }
3006 error_stream << "\n";
3007 error_stream << "The coordinates of its "
3009 << " master nodes are: \n";
3010 for (unsigned k = 0; k < n_master_replace; k++)
3011 {
3013 existing_node_pt->hanging_pt(i_cont)
3014 ->master_node_pt(k);
3015 unsigned ndim = master_nod_pt->ndim();
3016 for (unsigned i = 0; i < ndim; i++)
3017 {
3018 error_stream << master_nod_pt->x(i) << " ";
3019 }
3020 error_stream << "\n";
3021 }
3022 }
3023
3024 {
3026 << "Nodal coordinates of node to be replaced:";
3027 unsigned ndim = finite_ext_el_pt->node_pt(j)->ndim();
3028 for (unsigned i = 0; i < ndim; i++)
3029 {
3031 << " ";
3032 }
3033 error_stream << "\n";
3034 error_stream << "The coordinates of its "
3035 << n_master_orig
3036 << " master nodes are: \n";
3037 for (unsigned k = 0; k < n_master_orig; k++)
3038 {
3041 ->master_node_pt(k);
3042 unsigned ndim = master_nod_pt->ndim();
3043 for (unsigned i = 0; i < ndim; i++)
3044 {
3045 error_stream << master_nod_pt->x(i) << " ";
3046 }
3047 error_stream << "\n";
3048 }
3049 }
3050
3051
3052 throw OomphLibError(error_stream.str(),
3055 }
3056 }
3057 }
3058#endif
3059 // ...and point to the existing one
3061 }
3062 // If it doesn't add it to the list of existing ones
3063 else
3064 {
3066 nod_pt;
3067 node_done[nod_pt] = true;
3068 }
3069 }
3070
3071
3072 // Do the same for any master nodes of that (possibly replaced)
3073 // node
3074 if (dynamic_cast<RefineableElement*>(finite_ext_el_pt) != 0)
3075 {
3077 dynamic_cast<RefineableElement*>(finite_ext_el_pt)
3078 ->ncont_interpolated_values();
3079 for (int i_cont = -1; i_cont < n_cont_inter_values; i_cont++)
3080 {
3082 {
3083 HangInfo* hang_pt =
3085 unsigned n_master = hang_pt->nmaster();
3086 for (unsigned m = 0; m < n_master; m++)
3087 {
3088 Node* master_nod_pt = hang_pt->master_node_pt(m);
3089 unsigned n_val = master_nod_pt->nvalue();
3091 for (unsigned i_val = 0; i_val < n_val; i_val++)
3092 {
3094 if (eqn_no >= 0)
3095 {
3097 break;
3098 }
3099 }
3100
3101 // If we haven't found a non-negative eqn number check
3102 // eqn numbers associated with solid data (if any)
3104 {
3106 dynamic_cast<SolidNode*>(master_nod_pt);
3107 if (solid_master_nod_pt != 0)
3108 {
3109 // Loop over values stored at node (if any) to find
3110 // the first non-negative eqn number
3111 unsigned n_val =
3112 solid_master_nod_pt->variable_position_pt()
3113 ->nvalue();
3114 for (unsigned i_val = 0; i_val < n_val; i_val++)
3115 {
3116 int eqn_no =
3117 solid_master_nod_pt->variable_position_pt()
3118 ->eqn_number(i_val);
3119 if (eqn_no >= 0)
3120 {
3122 eqn_no + 1;
3123 break;
3124 }
3125 }
3126 }
3127 }
3128
3129 // Identified which node we're dealing with via
3130 // first non-negative global eqn number (if there
3131 // is none, everything is pinned and we don't give a
3132 // damn...)
3134 {
3137
3138 // Does this node already exist?
3139 if (existing_node_pt != 0)
3140 {
3141 // Record that we're about to cull one
3143
3144 // It's a duplicate, so store the duplicated one for
3145 // later killing...
3147
3149 {
3150 // Remove node from all boundaries
3151 std::set<unsigned>* boundaries_pt;
3152 duplicated_node_pt->get_boundaries_pt(
3154 if (boundaries_pt != 0)
3155 {
3156 for (std::set<unsigned>::iterator it =
3157 (*boundaries_pt).begin();
3158 it != (*boundaries_pt).end();
3159 it++)
3160 {
3163 }
3164 }
3165
3167 unsigned i_proc = unsigned(iproc);
3170 }
3171
3172 // Weight of the original node
3173 double m_weight = hang_pt->master_weight(m);
3174
3175
3176#ifdef PARANOID
3177 // Sanity check: setting replacement master
3178 // node for non-hanging node? Sign of really
3179 // f***ed up code.
3181 if (!tmp_nod_pt->is_hanging(i_cont))
3182 {
3183 std::ostringstream error_stream;
3185 << "About to re-set master for i_cont= " << i_cont
3186 << " for external node (with proc " << iproc
3187 << " )" << tmp_nod_pt << " at ";
3188 unsigned n = tmp_nod_pt->ndim();
3189 for (unsigned jj = 0; jj < n; jj++)
3190 {
3191 error_stream << tmp_nod_pt->x(jj) << " ";
3192 }
3194 << " which is not hanging --> About to die!"
3195 << "Outputting offending element into oomph-info "
3196 << "stream. \n\n";
3197 oomph_info << "\n\n";
3199 oomph_info << "\n\n";
3200 oomph_info.stream_pt()->flush();
3201 throw OomphLibError(error_stream.str(),
3204 }
3205#endif
3206
3207
3208 // And re-set master
3212 }
3213 // If it doesn't, add it to the list of existing ones
3214 else
3215 {
3219 node_done[master_nod_pt] = true;
3220 }
3221 }
3222 } // End of loop over master nodes
3223 } // end of hanging
3224 } // end of loop over continously interpolated variables
3225 } // end refineable element (with potentially hanging node
3226
3227 } // end loop over nodes on external halo elements
3228
3229 } // End of check for finite element
3230
3231 } // end loop over external halo elements
3232 }
3233 } // end loop over processors
3234
3235
3236 // Now kill all the deleted nodes
3237 for (std::set<Node*>::iterator it = killed_nodes.begin();
3238 it != killed_nodes.end();
3239 it++)
3240 {
3241 delete (*it);
3242 }
3243
3244
3245 // oomph_info << "Number of nonzero entries in global_node_pt: "
3246 // << global_node_pt.size() << std::endl;
3247
3248 // // Time it...
3249 // // Taken out again by MH -- clutters up output
3250 // if (Global_timings::Doc_comprehensive_timings)
3251 // {
3252 // double t_end = TimingHelpers::timer();
3253 // oomph_info
3254 // << "Total time for Problem::remove_duplicate_data: "
3255 // << t_end-t_start << std::endl;
3256 // }
3257 }
3258
3259
3260 //========================================================================
3261 /// Consolidate external halo node storage by removing nulled out
3262 /// pointers in external halo and haloed schemes for all meshes.
3263 //========================================================================
3265 {
3266 // Do we have submeshes?
3267 unsigned n_mesh_loop = 1;
3268 unsigned nmesh = nsub_mesh();
3269 if (nmesh > 0)
3270 {
3271 n_mesh_loop = nmesh;
3272 }
3273
3274 // Storage for number of processors and current processor
3275 int n_proc = this->communicator_pt()->nproc();
3276 int my_rank = this->communicator_pt()->my_rank();
3277
3278 // If only one processor then return
3279 if (n_proc == 1)
3280 {
3281 return;
3282 }
3283
3284 // Loop over all (other) processors and store index of any nulled-out
3285 // external halo nodes in storage scheme.
3286
3287 // Data to be sent to each processor
3289
3290 // Storage for all values to be sent to all processors
3292
3293 // Start location within send_data for data to be sent to each processor
3295
3296 // Check missing ones
3297 for (int domain = 0; domain < n_proc; domain++)
3298 {
3299 // Set the offset for the current processor
3301
3302 // Don't bother to do anything if the processor in the loop is the
3303 // current processor
3304 if (domain != my_rank)
3305 {
3306 // Deal with sub-meshes one-by-one if required
3307 Mesh* my_mesh_pt = 0;
3308
3309 // Loop over submeshes
3310 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3311 {
3312 if (nmesh == 0)
3313 {
3314 my_mesh_pt = mesh_pt();
3315 }
3316 else
3317 {
3319 }
3320
3321 // Make backup of external halo node pointers with this domain
3322 Vector<Node*> backup_pt(my_mesh_pt->external_halo_node_pt(domain));
3323
3324 // How many do we have currently?
3325 unsigned nnod = backup_pt.size();
3326
3327 // Prepare storage for updated halo nodes
3330
3331 // Loop over external halo nodes with this domain
3332 for (unsigned j = 0; j < nnod; j++)
3333 {
3334 // Get pointer to node
3335 Node* nod_pt = backup_pt[j];
3336
3337 // Has it been nulled out?
3338 if (nod_pt == 0)
3339 {
3340 // Save index of nulled out one
3341 send_data.push_back(j);
3342 }
3343 else
3344 {
3345 // Still alive: Copy across
3347 }
3348 }
3349
3350 // Set new external halo node vector
3351 my_mesh_pt->set_external_halo_node_pt(domain,
3353
3354 // End of data for this mesh
3355 send_data.push_back(-1);
3356
3357 } // end of loop over meshes
3358
3359 } // end skip own domain
3360
3361 // Find the number of data added to the vector
3363 }
3364
3365 // Storage for the number of data to be received from each processor
3367
3368 // Now send numbers of data to be sent between all processors
3369 MPI_Alltoall(&send_n[0],
3370 1,
3371 MPI_INT,
3372 &receive_n[0],
3373 1,
3374 MPI_INT,
3375 this->communicator_pt()->mpi_comm());
3376
3377
3378 // We now prepare the data to be received
3379 // by working out the displacements from the received data
3381 int receive_data_count = 0;
3382 for (int rank = 0; rank < n_proc; ++rank)
3383 {
3384 // Displacement is number of data received so far
3387 }
3388
3389 // Now resize the receive buffer for all data from all processors
3390 // Make sure that it has a size of at least one
3391 if (receive_data_count == 0)
3392 {
3394 }
3396
3397 // Make sure that the send buffer has size at least one
3398 // so that we don't get a segmentation fault
3399 if (send_data.size() == 0)
3400 {
3401 send_data.resize(1);
3402 }
3403
3404 // Now send the data between all the processors
3406 &send_n[0],
3408 MPI_INT,
3409 &receive_data[0],
3410 &receive_n[0],
3412 MPI_INT,
3413 this->communicator_pt()->mpi_comm());
3414
3415 // Now use the received data
3416 for (int send_rank = 0; send_rank < n_proc; send_rank++)
3417 {
3418 // Don't bother to do anything for the processor corresponding to the
3419 // current processor or if no data were received from this processor
3420 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
3421 {
3422 // Counter for the data within the large array
3424
3425 // Deal with sub-meshes one-by-one if required
3426 Mesh* my_mesh_pt = 0;
3427
3428 // Loop over submeshes
3429 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
3430 {
3431 if (nmesh == 0)
3432 {
3433 my_mesh_pt = mesh_pt();
3434 }
3435 else
3436 {
3438 }
3439
3440 // Make backup of external haloed node pointers with this domain
3442 my_mesh_pt->external_haloed_node_pt(send_rank);
3443
3444 // Unpack until we reach "end of data" indicator (-1) for this mesh
3445 while (true)
3446 {
3447 // Read next entry
3448 int next_one = receive_data[count++];
3449
3450 if (next_one == -1)
3451 {
3452 break;
3453 }
3454 else
3455 {
3456 // Null out the entry
3457 backup_pt[next_one] = 0;
3458 }
3459 }
3460
3461 // How many do we have currently?
3462 unsigned nnod = backup_pt.size();
3463
3464 // Prepare storage for updated haloed nodes
3467
3468 // Loop over external haloed nodes with this domain
3469 for (unsigned j = 0; j < nnod; j++)
3470 {
3471 // Get pointer to node
3472 Node* nod_pt = backup_pt[j];
3473
3474 // Has it been nulled out?
3475 if (nod_pt != 0)
3476 {
3477 // Still alive: Copy across
3479 }
3480 }
3481
3482 // Set new external haloed node vector
3483 my_mesh_pt->set_external_haloed_node_pt(send_rank,
3485 }
3486 }
3487
3488 } // End of data is received
3489 }
3490
3491#endif
3492
3493
3494 //=======================================================================
3495 /// Function that sets the values of the dofs in the object
3496 //======================================================================
3498 {
3499 const unsigned long n_dof = this->ndof();
3500#ifdef PARANOID
3501 if (n_dof != dofs.nrow())
3502 {
3503 std::ostringstream error_stream;
3504 error_stream << "Number of degrees of freedom in vector argument "
3505 << dofs.nrow() << "\n"
3506 << "does not equal number of degrees of freedom in problem "
3507 << n_dof;
3508 throw OomphLibError(
3510 }
3511#endif
3512 for (unsigned long l = 0; l < n_dof; l++)
3513 {
3514 *Dof_pt[l] = dofs[l];
3515 }
3516 }
3517
3518 /// Set history values of dofs
3519 void Problem::set_dofs(const unsigned& t, DoubleVector& dofs)
3520 {
3521#ifdef PARANOID
3522 if (distributed())
3523 {
3524 throw OomphLibError("Not designed for distributed problems",
3527 // might work if the dofs vector is distributed in the right way...
3528 }
3529#endif
3530
3531 // First deal with global data
3532 unsigned Nglobal_data = nglobal_data();
3533 for (unsigned i = 0; i < Nglobal_data; i++)
3534 {
3535 for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3536 {
3537 // For each data get the equation number and copy out the value.
3538 int eqn_number = Global_data_pt[i]->eqn_number(j);
3539 if (eqn_number >= 0)
3540 {
3541 Global_data_pt[i]->set_value(t, j, dofs[eqn_number]);
3542 }
3543 }
3544 }
3545
3546 // Next element internal data
3547 for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3548 {
3550 for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3551 {
3552 Data* d_pt = ele_pt->internal_data_pt(j);
3553 for (unsigned k = 0, nk = d_pt->nvalue(); k < nk; k++)
3554 {
3555 int eqn_number = d_pt->eqn_number(k);
3556 if (eqn_number >= 0)
3557 {
3558 d_pt->set_value(t, k, dofs[eqn_number]);
3559 }
3560 }
3561 }
3562 }
3563
3564 // Now the nodes
3565 for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3566 {
3567 Node* node_pt = mesh_pt()->node_pt(i);
3568 for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3569 {
3570 // For each node get the equation number and copy out the value.
3571 int eqn_number = node_pt->eqn_number(j);
3572 if (eqn_number >= 0)
3573 {
3574 node_pt->set_value(t, j, dofs[eqn_number]);
3575 }
3576 }
3577 }
3578 }
3579
3580
3581 /// Set history values of dofs from the type of vector stored in
3582 /// problem::Dof_pt.
3583 void Problem::set_dofs(const unsigned& t, Vector<double*>& dof_pt)
3584 {
3585#ifdef PARANOID
3586 if (distributed())
3587 {
3588 throw OomphLibError("Not implemented for distributed problems!",
3591 }
3592#endif
3593
3594 // If we have any spine meshes I think there might be more degrees
3595 // of freedom there. I don't use them though so I'll let someone who
3596 // knows what they are doing handle it. --David Shepherd
3597
3598 // First deal with global data
3599 unsigned Nglobal_data = nglobal_data();
3600 for (unsigned i = 0; i < Nglobal_data; i++)
3601 {
3602 for (unsigned j = 0, nj = Global_data_pt[i]->nvalue(); j < nj; j++)
3603 {
3604 // For each data get the equation number and copy in the value.
3605 int eqn_number = Global_data_pt[i]->eqn_number(j);
3606 if (eqn_number >= 0)
3607 {
3608 Global_data_pt[i]->set_value(t, j, *(dof_pt[eqn_number]));
3609 }
3610 }
3611 }
3612
3613 // Now the mesh data
3614 // nodes
3615 for (unsigned i = 0, ni = mesh_pt()->nnode(); i < ni; i++)
3616 {
3617 Node* node_pt = mesh_pt()->node_pt(i);
3618 for (unsigned j = 0, nj = node_pt->nvalue(); j < nj; j++)
3619 {
3620 // For each node get the equation number and copy in the value.
3621 int eqn_number = node_pt->eqn_number(j);
3622 if (eqn_number >= 0)
3623 {
3624 node_pt->set_value(t, j, *(dof_pt[eqn_number]));
3625 }
3626 }
3627 }
3628
3629 // and non-nodal data inside elements
3630 for (unsigned i = 0, ni = mesh_pt()->nelement(); i < ni; i++)
3631 {
3633 for (unsigned j = 0, nj = ele_pt->ninternal_data(); j < nj; j++)
3634 {
3636 // For each node get the equation number and copy in the value.
3637 int eqn_number = data_pt->eqn_number(j);
3638 if (eqn_number >= 0)
3639 {
3640 data_pt->set_value(t, j, *(dof_pt[eqn_number]));
3641 }
3642 }
3643 }
3644 }
3645
3646
3647 //===================================================================
3648 /// Function that adds the values to the dofs
3649 //==================================================================
3650 void Problem::add_to_dofs(const double& lambda,
3652 {
3653 const unsigned long n_dof = this->ndof();
3654 for (unsigned long l = 0; l < n_dof; l++)
3655 {
3656 *Dof_pt[l] += lambda * increment_dofs[l];
3657 }
3658 }
3659
3660
3661 //=========================================================================
3662 /// Return the residual vector multiplied by the inverse mass matrix
3663 /// Virtual so that it can be overloaded for mpi problems
3664 //=========================================================================
3666 {
3667 // This function does not make sense for assembly handlers other than the
3668 // default, so complain if we try to call it with another handler
3669
3670#ifdef PARANOID
3671 // If we are not the default, then complain
3673 {
3674 std::ostringstream error_stream;
3675 error_stream << "The function get_inverse_mass_matrix_times_residuals() "
3676 "can only be\n"
3677 << "used with the default assembly handler\n\n";
3678 throw OomphLibError(
3680 }
3681#endif
3682
3683 // Find the number of degrees of freedom in the problem
3684 const unsigned n_dof = this->ndof();
3685
3686 // Resize the vector
3687 LinearAlgebraDistribution dist(this->communicator_pt(), n_dof, false);
3688 Mres.build(&dist, 0.0);
3689
3690 // If we have discontinuous formulation
3691 // We can invert the mass matrix element by element
3693 {
3694 // Loop over the elements and get their residuals
3695 const unsigned n_element = Problem::mesh_pt()->nelement();
3697 for (unsigned e = 0; e < n_element; e++)
3698 {
3699 // Cache the element
3700 DGElement* const elem_pt =
3701 dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
3702
3703 // Find the elemental inverse mass matrix times residuals
3704 const unsigned n_el_dofs = elem_pt->ndof();
3705 elem_pt->get_inverse_mass_matrix_times_residuals(element_Mres);
3706
3707 // Add contribution to global matrix
3708 for (unsigned i = 0; i < n_el_dofs; i++)
3709 {
3711 }
3712 }
3713 }
3714 // Otherwise it's continous and we must invert the full
3715 // mass matrix via a global linear solve.
3716 else
3717 {
3718 // Now do the linear solve -- recycling Mass matrix if requested
3719 // If we already have the factorised mass matrix, then resolve
3721 {
3723 {
3724 oomph_info << "Not recomputing Mass Matrix " << std::endl;
3725 }
3726
3727 // Get the residuals
3729 this->get_residuals(residuals);
3730
3731 // Resolve the linear system
3733 residuals, Mres);
3734 }
3735 // Otherwise solve for the first time
3736 else
3737 {
3738 // If we wish to reuse the mass matrix, then enable resolve
3740 {
3742 {
3743 oomph_info << "Enabling resolve in explicit timestep" << std::endl;
3744 }
3746 ->enable_resolve();
3747 }
3748
3749 // Use a custom assembly handler to assemble and invert the mass matrix
3750
3751 // Store the old assembly handler
3753 // Set the assembly handler to the explicit timestep handler
3755
3756 // Solve the linear system
3758 Mres);
3759 // The mass matrix has now been computed
3761
3762 // Delete the Explicit Timestep handler
3763 delete this->assembly_handler_pt();
3764 // Reset the assembly handler to the original handler
3765 this->assembly_handler_pt() = old_assembly_handler_pt;
3766 }
3767 }
3768 }
3769
3771 {
3772 // Loop over timesteppers: make them (temporarily) steady and store their
3773 // is_steady status.
3774 unsigned n_time_steppers = this->ntime_stepper();
3775 std::vector<bool> was_steady(n_time_steppers);
3776 for (unsigned i = 0; i < n_time_steppers; i++)
3777 {
3780 }
3781
3782 // Calculate f using the residual/jacobian machinary.
3784
3785 // Reset the is_steady status of all timesteppers that weren't already
3786 // steady when we came in here and reset their weights
3787 for (unsigned i = 0; i < n_time_steppers; i++)
3788 {
3789 if (!was_steady[i])
3790 {
3792 }
3793 }
3794 }
3795
3796
3797 //================================================================
3798 /// Get the total residuals Vector for the problem
3799 //================================================================
3801 {
3802 // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
3803 // this means MPI_Helpers::init() has been called. This could happen on a
3804 // code compiled with MPI but run serially; in this instance the
3805 // get_residuals function still works on one processor.
3806 //
3807 // Secondly, if a code has been compiled with MPI, but MPI_Helpers::init()
3808 // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
3809 // and the code calls...
3810 //
3811 // Thirdly, the serial version (compiled by all, but only run when compiled
3812 // with MPI if MPI_Helpers::MPI_has_been_initialised=false
3813
3814 // Check that the residuals has the correct number of rows if it has been
3815 // setup
3816#ifdef PARANOID
3817 if (residuals.built())
3818 {
3819 if (residuals.distribution_pt()->nrow() != this->ndof())
3820 {
3821 std::ostringstream error_stream;
3822 error_stream << "The distribution of the residuals vector does not "
3823 "have the correct\n"
3824 << "number of global rows\n";
3825
3826 throw OomphLibError(
3828 }
3829 }
3830#endif
3831
3832 // Determine the distribution for the residuals vector
3833 // IF the vector has distribution setup then use that
3834 // ELSE determine the distribution based on the
3835 // distributed_matrix_distribution enum
3837 if (residuals.built())
3838 {
3839 dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
3840 }
3841 else
3842 {
3844 }
3845
3846 // Locally cache pointer to assembly handler
3848
3849 // Build and zero the residuals
3850 residuals.build(dist_pt, 0.0);
3851
3852 // Serial (or one processor case)
3853#ifdef OOMPH_HAS_MPI
3854 if (this->communicator_pt()->nproc() == 1)
3855 {
3856#endif // OOMPH_HAS_MPI
3857 // Loop over all the elements
3858 unsigned long Element_pt_range = Mesh_pt->nelement();
3859 for (unsigned long e = 0; e < Element_pt_range; e++)
3860 {
3861 // Get the pointer to the element
3863 // Find number of dofs in the element
3865 // Set up an array
3867 // Fill the array
3869 // Now loop over the dofs and assign values to global Vector
3870 for (unsigned l = 0; l < n_element_dofs; l++)
3871 {
3874 }
3875 }
3876 // Otherwise parallel case
3877#ifdef OOMPH_HAS_MPI
3878 }
3879 else
3880 {
3881 // Store the current assembly handler
3883 // Create a new assembly handler that only assembles the residuals
3886
3887 // Setup memory for parallel sparse assemble
3888 // No matrix so all size zero
3889 Vector<int*> column_index;
3890 Vector<int*> row_start;
3891 Vector<double*> value;
3892 Vector<unsigned> nnz;
3893 // One set of residuals of sizer one
3895
3896 // Call the parallel sparse assemble, that should only assemble residuals
3898 dist_pt, column_index, row_start, value, nnz, res);
3899 // Fill in the residuals data
3900 residuals.set_external_values(res[0], true);
3901
3902 // Delete new assembly handler
3903 delete Assembly_handler_pt;
3904 // Reset the assembly handler to the original
3906 }
3907#endif
3908
3909 // Delete the distribution
3910 delete dist_pt;
3911 }
3912
3913 //=============================================================================
3914 /// Get the fully assembled residual vector and Jacobian matrix
3915 /// in dense storage. The DoubleVector residuals returned will be
3916 /// non-distributed. If on calling this method the DoubleVector residuals is
3917 /// setup then it must be non-distributed and of the correct length.
3918 /// The matrix type DenseDoubleMatrix is not distributable and therefore
3919 /// the residual vector is also assumed to be non distributable.
3920 //=============================================================================
3922 DenseDoubleMatrix& jacobian)
3923 {
3924 // get the number of degrees of freedom
3925 unsigned n_dof = ndof();
3926
3927#ifdef PARANOID
3928 // PARANOID checks : if the distribution of residuals is setup then it must
3929 // must not be distributed, have the right number of rows, and the same
3930 // communicator as the problem
3931 if (residuals.built())
3932 {
3933 if (residuals.distribution_pt()->distributed())
3934 {
3935 std::ostringstream error_stream;
3937 << "If the DoubleVector residuals is setup then it must not "
3938 << "be distributed.";
3939 throw OomphLibError(
3941 }
3942 if (residuals.distribution_pt()->nrow() != n_dof)
3943 {
3944 std::ostringstream error_stream;
3946 << "If the DoubleVector residuals is setup then it must have"
3947 << " the correct number of rows";
3948 throw OomphLibError(
3950 }
3951 if (!(*Communicator_pt ==
3952 *residuals.distribution_pt()->communicator_pt()))
3953 {
3954 std::ostringstream error_stream;
3956 << "If the DoubleVector residuals is setup then it must have"
3957 << " the same communicator as the problem.";
3958 throw OomphLibError(
3960 }
3961 }
3962#endif
3963
3964 // set the residuals distribution if it is not setup
3965 if (!residuals.built())
3966 {
3968 residuals.build(&dist, 0.0);
3969 }
3970 // else just zero the residuals
3971 else
3972 {
3973 residuals.initialise(0.0);
3974 }
3975
3976 // Resize the matrices -- this cannot always be done externally
3977 // because get_jacobian exists in many different versions for
3978 // different storage formats -- resizing a CC or CR matrix doesn't
3979 // make sense.
3980
3981 // resize the jacobian
3982 jacobian.resize(n_dof, n_dof);
3983 jacobian.initialise(0.0);
3984
3985 // Locally cache pointer to assembly handler
3987
3988 // Loop over all the elements
3989 unsigned long n_element = Mesh_pt->nelement();
3990 for (unsigned long e = 0; e < n_element; e++)
3991 {
3992 // Get the pointer to the element
3994 // Find number of dofs in the element
3996 // Set up an array
3998 // Set up a matrix
4000 // Fill the array
4003 // Now loop over the dofs and assign values to global Vector
4004 for (unsigned l = 0; l < n_element_dofs; l++)
4005 {
4006 unsigned long eqn_number = assembly_handler_pt->eqn_number(elem_pt, l);
4007 residuals[eqn_number] += element_residuals[l];
4008 for (unsigned l2 = 0; l2 < n_element_dofs; l2++)
4009 {
4010 jacobian(eqn_number, assembly_handler_pt->eqn_number(elem_pt, l2)) +=
4012 }
4013 }
4014 }
4015 }
4016
4017 //=============================================================================
4018 /// Return the fully-assembled Jacobian and residuals for the problem,
4019 /// in the case where the Jacobian matrix is in a distributable
4020 /// row compressed storage format.
4021 /// 1. If the distribution of the jacobian and residuals is setup then, they
4022 /// will be returned with that distribution.
4023 /// Note. the jacobian and residuals must have the same distribution.
4024 /// 2. If the distribution of the jacobian and residuals are not setup then
4025 /// their distribution will computed based on:
4026 /// Distributed_problem_matrix_distribution.
4027 //=============================================================================
4029 {
4030 // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4031 // this means MPI_Helpers::setup() has been called. This could happen on a
4032 // code compiled with MPI but run serially; in this instance the
4033 // get_residuals function still works on one processor.
4034 //
4035 // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4036 // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4037 // and the code calls...
4038 //
4039 // Thirdly, the serial version (compiled by all, but only run when compiled
4040 // with MPI if MPI_Helpers::MPI_has_been_initialised=false
4041 //
4042 // The only case where an MPI code cannot run serially at present
4043 // is one where the distribute function is used (i.e. METIS is called)
4044
4045 // Allocate storage for the matrix entries
4046 // The generalised Vector<Vector<>> structure is required
4047 // for the most general interface to sparse_assemble() which allows
4048 // the assembly of multiple matrices at once.
4049 Vector<int*> column_index(1);
4050 Vector<int*> row_start(1);
4051 Vector<double*> value(1);
4052 Vector<unsigned> nnz(1);
4053
4054#ifdef PARANOID
4055 // PARANOID checks that the distribution of the jacobian matches that of the
4056 // residuals (if they are setup) and that they have the right number of rows
4057 if (residuals.built() && jacobian.distribution_built())
4058 {
4059 if (!(*residuals.distribution_pt() == *jacobian.distribution_pt()))
4060 {
4061 std::ostringstream error_stream;
4062 error_stream << "The distribution of the residuals must "
4063 << "be the same as the distribution of the jacobian.";
4064 throw OomphLibError(
4066 }
4067 if (jacobian.distribution_pt()->nrow() != this->ndof())
4068 {
4069 std::ostringstream error_stream;
4071 << "The distribution of the jacobian and residuals does not"
4072 << "have the correct number of global rows.";
4073 throw OomphLibError(
4075 }
4076 }
4077 else if (residuals.built() != jacobian.distribution_built())
4078 {
4079 std::ostringstream error_stream;
4080 error_stream << "The distribution of the jacobian and residuals must "
4081 << "both be setup or both not setup";
4082 throw OomphLibError(
4084 }
4085#endif
4086
4087
4088 // Allocate generalised storage format for passing to sparse_assemble()
4090
4091 // determine the distribution for the jacobian.
4092 // IF the jacobian has distribution setup then use that
4093 // ELSE determine the distribution based on the
4094 // distributed_matrix_distribution enum
4096 if (jacobian.distribution_built())
4097 {
4099 }
4100 else
4101 {
4103 }
4104
4105
4106 // The matrix is in compressed row format
4107 bool compressed_row_flag = true;
4108
4109#ifdef OOMPH_HAS_MPI
4110 //
4111 if (Communicator_pt->nproc() == 1)
4112 {
4113#endif
4115 column_index, row_start, value, nnz, res, compressed_row_flag);
4116 jacobian.build(dist_pt);
4117 jacobian.build_without_copy(
4118 dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4119 residuals.build(dist_pt, 0.0);
4120 residuals.set_external_values(res[0], true);
4121#ifdef OOMPH_HAS_MPI
4122 }
4123 else
4124 {
4125 if (dist_pt->distributed())
4126 {
4128 dist_pt, column_index, row_start, value, nnz, res);
4129 jacobian.build(dist_pt);
4130 jacobian.build_without_copy(
4131 dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4132 residuals.build(dist_pt, 0.0);
4133 residuals.set_external_values(res[0], true);
4134 }
4135 else
4136 {
4140 temp_dist_pt, column_index, row_start, value, nnz, res);
4141 jacobian.build(temp_dist_pt);
4142 jacobian.build_without_copy(
4143 dist_pt->nrow(), nnz[0], value[0], column_index[0], row_start[0]);
4144 jacobian.redistribute(dist_pt);
4145 residuals.build(temp_dist_pt, 0.0);
4146 residuals.set_external_values(res[0], true);
4147 residuals.redistribute(dist_pt);
4148 delete temp_dist_pt;
4149 }
4150 }
4151#endif
4152
4153 // clean up dist_pt and residuals_vector pt
4154 delete dist_pt;
4155 }
4156
4157 //=============================================================================
4158 /// Return the fully-assembled Jacobian and residuals for the problem,
4159 /// in the case when the jacobian matrix is in column-compressed storage
4160 /// format.
4161 //=============================================================================
4163 {
4164 // Three different cases; if MPI_Helpers::MPI_has_been_initialised=true
4165 // this means MPI_Helpers::setup() has been called. This could happen on a
4166 // code compiled with MPI but run serially; in this instance the
4167 // get_residuals function still works on one processor.
4168 //
4169 // Secondly, if a code has been compiled with MPI, but MPI_Helpers::setup()
4170 // has not been called, then MPI_Helpers::MPI_has_been_initialised=false
4171 // and the code calls...
4172 //
4173 // Thirdly, the serial version (compiled by all, but only run when compiled
4174 // with MPI if MPI_Helpers::MPI_has_been_5Binitialised=false
4175 //
4176 // The only case where an MPI code cannot run serially at present
4177 // is one where the distribute function is used (i.e. METIS is called)
4178
4179 // get the number of degrees of freedom
4180 unsigned n_dof = ndof();
4181
4182#ifdef PARANOID
4183 // PARANOID checks : if the distribution of residuals is setup then it must
4184 // must not be distributed, have the right number of rows, and the same
4185 // communicator as the problem
4186 if (residuals.built())
4187 {
4188 if (residuals.distribution_pt()->distributed())
4189 {
4190 std::ostringstream error_stream;
4192 << "If the DoubleVector residuals is setup then it must not "
4193 << "be distributed.";
4194 throw OomphLibError(
4196 }
4197 if (residuals.distribution_pt()->nrow() != n_dof)
4198 {
4199 std::ostringstream error_stream;
4201 << "If the DoubleVector residuals is setup then it must have"
4202 << " the correct number of rows";
4203 throw OomphLibError(
4205 }
4206 if (!(*Communicator_pt ==
4207 *residuals.distribution_pt()->communicator_pt()))
4208 {
4209 std::ostringstream error_stream;
4211 << "If the DoubleVector residuals is setup then it must have"
4212 << " the same communicator as the problem.";
4213 throw OomphLibError(
4215 }
4216 }
4217#endif
4218
4219 // Allocate storage for the matrix entries
4220 // The generalised Vector<Vector<>> structure is required
4221 // for the most general interface to sparse_assemble() which allows
4222 // the assembly of multiple matrices at once.
4223 Vector<int*> row_index(1);
4224 Vector<int*> column_start(1);
4225 Vector<double*> value(1);
4226
4227 // Allocate generalised storage format for passing to sparse_assemble()
4229
4230 // allocate storage for the number of non-zeros in each matrix
4231 Vector<unsigned> nnz(1);
4232
4233 // The matrix is in compressed column format
4234 bool compressed_row_flag = false;
4235
4236 // get the distribution for the residuals
4238 if (!residuals.built())
4239 {
4240 dist_pt =
4241 new LinearAlgebraDistribution(Communicator_pt, this->ndof(), false);
4242 }
4243 else
4244 {
4245 dist_pt = new LinearAlgebraDistribution(residuals.distribution_pt());
4246 }
4247
4248#ifdef OOMPH_HAS_MPI
4249 if (communicator_pt()->nproc() == 1)
4250 {
4251#endif
4253 row_index, column_start, value, nnz, res, compressed_row_flag);
4254 jacobian.build_without_copy(
4255 value[0], row_index[0], column_start[0], nnz[0], n_dof, n_dof);
4256 residuals.build(dist_pt, 0.0);
4257 residuals.set_external_values(res[0], true);
4258#ifdef OOMPH_HAS_MPI
4259 }
4260 else
4261 {
4262 std::ostringstream error_stream;
4263 error_stream << "Cannot assemble a CCDoubleMatrix Jacobian on more "
4264 << "than one processor.";
4265 throw OomphLibError(
4267 }
4268#endif
4269
4270 // clean up
4271 delete dist_pt;
4272 }
4273
4274
4275 //===================================================================
4276 /// Set all pinned values to zero.
4277 /// Used to set boundary conditions to be homogeneous in the copy
4278 /// of the problem used in adaptive bifurcation tracking
4279 /// (ALH: TEMPORARY HACK, WILL BE FIXED)
4280 //==================================================================
4282 {
4283 // NOTE THIS DOES NOT ZERO ANY SPINE DATA, but otherwise everything else
4284 // should be zeroed
4285
4286 // Zero any pinned global Data
4287 const unsigned n_global_data = nglobal_data();
4288 for (unsigned i = 0; i < n_global_data; i++)
4289 {
4291 const unsigned n_value = local_data_pt->nvalue();
4292 for (unsigned j = 0; j < n_value; j++)
4293 {
4294 // If the data value is pinned set the value to zero
4295 if (local_data_pt->is_pinned(j))
4296 {
4297 local_data_pt->set_value(j, 0.0);
4298 }
4299 }
4300 }
4301
4302 // Loop over the submeshes:
4303 const unsigned n_sub_mesh = Sub_mesh_pt.size();
4304 if (n_sub_mesh == 0)
4305 {
4306 // Loop over the nodes in the element
4307 const unsigned n_node = Mesh_pt->nnode();
4308 for (unsigned n = 0; n < n_node; n++)
4309 {
4310 Node* const local_node_pt = Mesh_pt->node_pt(n);
4311 const unsigned n_value = local_node_pt->nvalue();
4312 for (unsigned j = 0; j < n_value; j++)
4313 {
4314 // If the data value is pinned set the value to zero
4315 if (local_node_pt->is_pinned(j))
4316 {
4317 local_node_pt->set_value(j, 0.0);
4318 }
4319 }
4320
4321 // Try to cast to a solid node
4323 dynamic_cast<SolidNode*>(local_node_pt);
4324 // If we are successful
4326 {
4327 // Find the dimension of the node
4328 const unsigned n_dim = local_solid_node_pt->ndim();
4329 // Find number of positions
4330 const unsigned n_position_type =
4331 local_solid_node_pt->nposition_type();
4332
4333 for (unsigned k = 0; k < n_position_type; k++)
4334 {
4335 for (unsigned i = 0; i < n_dim; i++)
4336 {
4337 // If the generalised position is pinned,
4338 // set the value to zero
4339 if (local_solid_node_pt->position_is_pinned(k, i))
4340 {
4341 local_solid_node_pt->x_gen(k, i) = 0.0;
4342 }
4343 }
4344 }
4345 }
4346 }
4347
4348 // Now loop over the element's and zero the internal data
4349 const unsigned n_element = Mesh_pt->nelement();
4350 for (unsigned e = 0; e < n_element; e++)
4351 {
4353 const unsigned n_internal = local_element_pt->ninternal_data();
4354 for (unsigned i = 0; i < n_internal; i++)
4355 {
4357 const unsigned n_value = local_data_pt->nvalue();
4358 for (unsigned j = 0; j < n_value; j++)
4359 {
4360 // If the data value is pinned set the value to zero
4361 if (local_data_pt->is_pinned(j))
4362 {
4363 local_data_pt->set_value(j, 0.0);
4364 }
4365 }
4366 }
4367 } // End of loop over elements
4368 }
4369 else
4370 {
4371 // Alternatively loop over all sub meshes
4372 for (unsigned m = 0; m < n_sub_mesh; m++)
4373 {
4374 // Loop over the nodes in the element
4375 const unsigned n_node = Sub_mesh_pt[m]->nnode();
4376 for (unsigned n = 0; n < n_node; n++)
4377 {
4379 const unsigned n_value = local_node_pt->nvalue();
4380 for (unsigned j = 0; j < n_value; j++)
4381 {
4382 // If the data value is pinned set the value to zero
4383 if (local_node_pt->is_pinned(j))
4384 {
4385 local_node_pt->set_value(j, 0.0);
4386 }
4387 }
4388
4389 // Try to cast to a solid node
4391 dynamic_cast<SolidNode*>(local_node_pt);
4392 // If we are successful
4394 {
4395 // Find the dimension of the node
4396 const unsigned n_dim = local_solid_node_pt->ndim();
4397 // Find number of positions
4398 const unsigned n_position_type =
4399 local_solid_node_pt->nposition_type();
4400
4401 for (unsigned k = 0; k < n_position_type; k++)
4402 {
4403 for (unsigned i = 0; i < n_dim; i++)
4404 {
4405 // If the generalised position is pinned,
4406 // set the value to zero
4407 if (local_solid_node_pt->position_is_pinned(k, i))
4408 {
4409 local_solid_node_pt->x_gen(k, i) = 0.0;
4410 }
4411 }
4412 }
4413 }
4414 }
4415
4416 // Now loop over the element's and zero the internal data
4417 const unsigned n_element = Sub_mesh_pt[m]->nelement();
4418 for (unsigned e = 0; e < n_element; e++)
4419 {
4421 Sub_mesh_pt[m]->element_pt(e);
4422 const unsigned n_internal = local_element_pt->ninternal_data();
4423 for (unsigned i = 0; i < n_internal; i++)
4424 {
4426 const unsigned n_value = local_data_pt->nvalue();
4427 for (unsigned j = 0; j < n_value; j++)
4428 {
4429 // If the data value is pinned set the value to zero
4430 if (local_data_pt->is_pinned(j))
4431 {
4432 local_data_pt->set_value(j, 0.0);
4433 }
4434 }
4435 }
4436 } // End of loop over elements
4437 }
4438 }
4439 }
4440
4441
4442 //=====================================================================
4443 /// This is a (private) helper function that is used to assemble system
4444 /// matrices in compressed row or column format
4445 /// and compute residual vectors.
4446 /// The default action is to assemble the jacobian matrix and
4447 /// residuals for the Newton method. The action can be
4448 /// overloaded at an elemental level by changing the default
4449 /// behaviour of the function Element::get_all_vectors_and_matrices().
4450 /// column_or_row_index: Column [or row] index of given entry
4451 /// row_or_column_start: Index of first entry for given row [or column]
4452 /// value : Vector of nonzero entries
4453 /// residuals : Residual vector
4454 /// compressed_row_flag: Bool flag to indicate if storage format is
4455 /// compressed row [if false interpretation of
4456 /// arguments is as stated in square brackets].
4457 /// We provide four different assembly methods, each with different
4458 /// memory requirements/execution speeds. The method is set by
4459 /// the public flag Problem::Sparse_assembly_method.
4460 //=====================================================================
4464 Vector<double*>& value,
4465 Vector<unsigned>& nnz,
4468 {
4469 // Choose the actual method
4470 switch (Sparse_assembly_method)
4471 {
4473
4477 value,
4478 nnz,
4479 residuals,
4481
4482 break;
4483
4485
4489 value,
4490 nnz,
4491 residuals,
4493
4494 break;
4495
4497
4500 value,
4501 nnz,
4502 residuals,
4504
4505 break;
4506
4508
4512 value,
4513 nnz,
4514 residuals,
4516
4517 break;
4518
4520
4524 value,
4525 nnz,
4526 residuals,
4528
4529 break;
4530
4531 default:
4532
4533 std::ostringstream error_stream;
4535 << "Error: Incorrect value for Problem::Sparse_assembly_method"
4536 << Sparse_assembly_method << std::endl
4537 << "It should be one of the enumeration Problem::Assembly_method"
4538 << std::endl;
4539 throw OomphLibError(
4541 }
4542 }
4543
4544
4545 //=====================================================================
4546 /// This is a (private) helper function that is used to assemble system
4547 /// matrices in compressed row or column format
4548 /// and compute residual vectors, using maps
4549 /// The default action is to assemble the jacobian matrix and
4550 /// residuals for the Newton method. The action can be
4551 /// overloaded at an elemental level by chaging the default
4552 /// behaviour of the function Element::get_all_vectors_and_matrices().
4553 /// column_or_row_index: Column [or row] index of given entry
4554 /// row_or_column_start: Index of first entry for given row [or column]
4555 /// value : Vector of nonzero entries
4556 /// residuals : Residual vector
4557 /// compressed_row_flag: Bool flag to indicate if storage format is
4558 /// compressed row [if false interpretation of
4559 /// arguments is as stated in square brackets].
4560 //=====================================================================
4564 Vector<double*>& value,
4565 Vector<unsigned>& nnz,
4568 {
4569 // Total number of elements
4570 const unsigned long n_elements = mesh_pt()->nelement();
4571
4572 // Default range of elements for distributed problems
4573 unsigned long el_lo = 0;
4574 unsigned long el_hi = n_elements - 1;
4575
4576#ifdef OOMPH_HAS_MPI
4577 // Otherwise just loop over a fraction of the elements
4578 // (This will either have been initialised in
4579 // Problem::set_default_first_and_last_element_for_assembly() or
4580 // will have been re-assigned during a previous assembly loop
4581 // Note that following the re-assignment only the entries
4582 // for the current processor are relevant.
4584 {
4587 }
4588#endif
4589
4590 // number of dofs
4591 unsigned ndof = this->ndof();
4592
4593 // Find the number of vectors to be assembled
4594 const unsigned n_vector = residuals.size();
4595
4596 // Find the number of matrices to be assembled
4597 const unsigned n_matrix = column_or_row_index.size();
4598
4599 // Locally cache pointer to assembly handler
4601
4602#ifdef OOMPH_HAS_MPI
4603 bool doing_residuals = false;
4604 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4605 {
4606 doing_residuals = true;
4607 }
4608#endif
4609
4610// Error check dimensions
4611#ifdef PARANOID
4613 {
4614 std::ostringstream error_stream;
4615 error_stream << "Error: " << std::endl
4616 << "row_or_column_start.size() "
4617 << row_or_column_start.size() << " does not equal "
4618 << "column_or_row_index.size() "
4619 << column_or_row_index.size() << std::endl;
4620 throw OomphLibError(
4622 }
4623
4624 if (value.size() != n_matrix)
4625 {
4626 std::ostringstream error_stream;
4628 << "Error in Problem::sparse_assemble_row_or_column_compressed "
4629 << std::endl
4630 << "value.size() " << value.size() << " does not equal "
4631 << "column_or_row_index.size() " << column_or_row_index.size()
4632 << std::endl
4633 << std::endl
4634 << std::endl;
4635 throw OomphLibError(
4637 }
4638#endif
4639
4640
4641 // The idea behind this sparse assembly routine is to use a vector of
4642 // maps for the entries in each row or column of the complete matrix.
4643 // The key for each map is the global row or column number and
4644 // the default comparison operator for integers means that each map
4645 // is ordered by the global row or column number. Thus, we need not
4646 // sort the maps, that happens at each insertion of a new entry. The
4647 // price we pay is that for large maps, inseration is not a
4648 // cheap operation. Hash maps can be used to increase the speed, but then
4649 // the ordering is lost and we would have to sort anyway. The solution if
4650 // speed is required is to use lists, see below.
4651
4652
4653 // Set up a vector of vectors of maps of entries of each matrix,
4654 // indexed by either the column or row. The entries of the vector for
4655 // each matrix correspond to all the rows or columns of that matrix.
4656 // The use of the map storage
4657 // scheme, with its implicit ordering on the first index, gives
4658 // a sparse ordered list of the entries in the given row or column.
4660 // Loop over the number of matrices being assembled and resize
4661 // each vector of maps to the number of rows or columns of the matrix
4662 for (unsigned m = 0; m < n_matrix; m++)
4663 {
4664 matrix_data_map[m].resize(ndof);
4665 }
4666
4667 // Resize the residuals vectors
4668 for (unsigned v = 0; v < n_vector; v++)
4669 {
4670 residuals[v] = new double[ndof];
4671 for (unsigned i = 0; i < ndof; i++)
4672 {
4673 residuals[v][i] = 0;
4674 }
4675 }
4676
4677
4678#ifdef OOMPH_HAS_MPI
4679
4680
4681 // Storage for assembly time for elements
4682 double t_assemble_start = 0.0;
4683
4684 // Storage for assembly times
4686 {
4688 }
4689
4690#endif
4691
4692 //----------------Assemble and populate the maps-------------------------
4693 {
4694 // Allocate local storage for the element's contribution to the
4695 // residuals vectors and system matrices of the size of the maximum
4696 // number of dofs in any element.
4697 // This means that the storage is only allocated (and deleted) once
4700
4701 // Loop over the elements for this processor
4702 for (unsigned long e = el_lo; e <= el_hi; e++)
4703 {
4704#ifdef OOMPH_HAS_MPI
4705 // Time it?
4707 {
4709 }
4710#endif
4711
4712 // Get the pointer to the element
4714
4715#ifdef OOMPH_HAS_MPI
4716 // Ignore halo elements
4717 if (!elem_pt->is_halo())
4718 {
4719#endif
4720
4721 // Find number of degrees of freedom in the element
4722 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
4723
4724 // Resize the storage for elemental jacobian and residuals
4725 for (unsigned v = 0; v < n_vector; v++)
4726 {
4727 el_residuals[v].resize(nvar);
4728 }
4729 for (unsigned m = 0; m < n_matrix; m++)
4730 {
4731 el_jacobian[m].resize(nvar);
4732 }
4733
4734 // Now get the residuals and jacobian for the element
4737
4738 //---------------Insert the values into the maps--------------
4739
4740 // Loop over the first index of local variables
4741 for (unsigned i = 0; i < nvar; i++)
4742 {
4743 // Get the local equation number
4744 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
4745
4746 // Add the contribution to the residuals
4747 for (unsigned v = 0; v < n_vector; v++)
4748 {
4749 // Fill in each residuals vector
4750 residuals[v][eqn_number] += el_residuals[v][i];
4751 }
4752
4753 // Now loop over the other index
4754 for (unsigned j = 0; j < nvar; j++)
4755 {
4756 // Get the number of the unknown
4758
4759 // Loop over the matrices
4760 for (unsigned m = 0; m < n_matrix; m++)
4761 {
4762 // Get the value of the matrix at this point
4763 double value = el_jacobian[m](i, j);
4764 // Only bother to add to the map if it's non-zero
4765 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
4766 {
4767 // If it's compressed row storage, then our vector of maps
4768 // is indexed by row (equation number)
4770 {
4771 // Add the data into the map using the unknown as the map
4772 // key
4773 matrix_data_map[m][eqn_number][unknown] += value;
4774 }
4775 // Otherwise it's compressed column storage and our vector is
4776 // indexed by column (the unknown)
4777 else
4778 {
4779 // Add the data into the map using the eqn_numbe as the map
4780 // key
4781 matrix_data_map[m][unknown][eqn_number] += value;
4782 }
4783 }
4784 } // End of loop over matrices
4785 }
4786 }
4787
4788#ifdef OOMPH_HAS_MPI
4789 } // endif halo element
4790#endif
4791
4792
4793#ifdef OOMPH_HAS_MPI
4794 // Time it?
4796 {
4799 }
4800#endif
4801
4802 } // End of loop over the elements
4803
4804 } // End of map assembly
4805
4806
4807#ifdef OOMPH_HAS_MPI
4808
4809 // Postprocess timing information and re-allocate distribution of
4810 // elements during subsequent assemblies.
4813 {
4815 }
4816
4817 // We have determined load balancing for current setup.
4818 // This can remain the same until assign_eqn_numbers() is called
4819 // again -- the flag is re-set to true there.
4821 {
4823 }
4824
4825#endif
4826
4827
4828 //-----------Finally we need to convert the beautiful map storage scheme
4829 //------------------------to the containers required by SuperLU
4830
4831 // Loop over the number of matrices
4832 for (unsigned m = 0; m < n_matrix; m++)
4833 {
4834 // Set the number of rows or columns
4835 row_or_column_start[m] = new int[ndof + 1];
4836 // Counter for the total number of entries in the storage scheme
4837 unsigned long entry_count = 0;
4839
4840 // first we compute the number of non-zeros
4841 nnz[m] = 0;
4842 for (unsigned long i_global = 0; i_global < ndof; i_global++)
4843 {
4844 nnz[m] += matrix_data_map[m][i_global].size();
4845 }
4846
4847 // and then resize the storage
4848 column_or_row_index[m] = new int[nnz[m]];
4849 value[m] = new double[nnz[m]];
4850
4851 // Now we merely loop over the number of rows or columns
4852 for (unsigned long i_global = 0; i_global < ndof; i_global++)
4853 {
4854 // Start index for the present row
4856 // If there are no entries in the map then skip the rest of the loop
4858 {
4859 continue;
4860 }
4861
4862 // Loop over all the entries in the map corresponding to the given
4863 // row or column. It will be ordered
4864
4865 for (std::map<unsigned, double>::iterator it =
4867 it != matrix_data_map[m][i_global].end();
4868 ++it)
4869 {
4870 // The first value is the column or row index
4872 // The second value is the actual data value
4873 value[m][entry_count] = it->second;
4874 // Increase the value of the counter
4875 entry_count++;
4876 }
4877 }
4878
4879 // Final entry in the row/column start vector
4881 } // End of the loop over the matrices
4882
4884 {
4885 oomph_info << "Pausing at end of sparse assembly." << std::endl;
4886 pause("Check memory usage now.");
4887 }
4888 }
4889
4890
4891 //=====================================================================
4892 /// This is a (private) helper function that is used to assemble system
4893 /// matrices in compressed row or column format
4894 /// and compute residual vectors using lists
4895 /// The default action is to assemble the jacobian matrix and
4896 /// residuals for the Newton method. The action can be
4897 /// overloaded at an elemental level by chaging the default
4898 /// behaviour of the function Element::get_all_vectors_and_matrices().
4899 /// column_or_row_index: Column [or row] index of given entry
4900 /// row_or_column_start: Index of first entry for given row [or column]
4901 /// value : Vector of nonzero entries
4902 /// residuals : Residual vector
4903 /// compressed_row_flag: Bool flag to indicate if storage format is
4904 /// compressed row [if false interpretation of
4905 /// arguments is as stated in square brackets].
4906 //=====================================================================
4910 Vector<double*>& value,
4911 Vector<unsigned>& nnz,
4914 {
4915 // Total number of elements
4916 const unsigned long n_elements = mesh_pt()->nelement();
4917
4918 // Default range of elements for distributed problems
4919 unsigned long el_lo = 0;
4920 unsigned long el_hi = n_elements - 1;
4921
4922#ifdef OOMPH_HAS_MPI
4923 // Otherwise just loop over a fraction of the elements
4924 // (This will either have been initialised in
4925 // Problem::set_default_first_and_last_element_for_assembly() or
4926 // will have been re-assigned during a previous assembly loop
4927 // Note that following the re-assignment only the entries
4928 // for the current processor are relevant.
4930 {
4933 }
4934#endif
4935
4936 // number of dofs
4937 unsigned ndof = this->ndof();
4938
4939 // Find the number of vectors to be assembled
4940 const unsigned n_vector = residuals.size();
4941
4942 // Find the number of matrices to be assembled
4943 const unsigned n_matrix = column_or_row_index.size();
4944
4945 // Locally cache pointer to assembly handler
4947
4948#ifdef OOMPH_HAS_MPI
4949 bool doing_residuals = false;
4950 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
4951 {
4952 doing_residuals = true;
4953 }
4954#endif
4955
4956// Error check dimensions
4957#ifdef PARANOID
4959 {
4960 std::ostringstream error_stream;
4961 error_stream << "Error: " << std::endl
4962 << "row_or_column_start.size() "
4963 << row_or_column_start.size() << " does not equal "
4964 << "column_or_row_index.size() "
4965 << column_or_row_index.size() << std::endl;
4966 throw OomphLibError(
4968 }
4969
4970 if (value.size() != n_matrix)
4971 {
4972 std::ostringstream error_stream;
4974 << "Error in Problem::sparse_assemble_row_or_column_compressed "
4975 << std::endl
4976 << "value.size() " << value.size() << " does not equal "
4977 << "column_or_row_index.size() " << column_or_row_index.size()
4978 << std::endl
4979 << std::endl
4980 << std::endl;
4981 throw OomphLibError(
4983 }
4984#endif
4985
4986 // The idea behind this sparse assembly routine is to use a vector of
4987 // lists for the entries in each row or column of the complete matrix.
4988 // The lists contain pairs of entries (global row/column number, value).
4989 // All non-zero contributions from each element are added to the lists.
4990 // We then sort each list by global row/column number and then combine
4991 // the entries corresponding to each row/column before adding to the
4992 // vectors column_or_row_index and value.
4993
4994 // Note the trade off for "fast assembly" is that we will require
4995 // more memory during the assembly phase. Then again, if we can
4996 // only just assemble the sparse matrix, we're in real trouble.
4997
4998 // Set up a vector of lists of paired entries of
4999 //(row/column index, jacobian matrix entry).
5000 // The entries of the vector correspond to all the rows or columns.
5001 // The use of the list storage scheme, should give fast insertion
5002 // and fast sorts later.
5004 n_matrix);
5005 // Loop over the number of matrices and resize
5006 for (unsigned m = 0; m < n_matrix; m++)
5007 {
5008 matrix_data_list[m].resize(ndof);
5009 }
5010
5011 // Resize the residuals vectors
5012 for (unsigned v = 0; v < n_vector; v++)
5013 {
5014 residuals[v] = new double[ndof];
5015 for (unsigned i = 0; i < ndof; i++)
5016 {
5017 residuals[v][i] = 0;
5018 }
5019 }
5020
5021#ifdef OOMPH_HAS_MPI
5022
5023
5024 // Storage for assembly time for elements
5025 double t_assemble_start = 0.0;
5026
5027 // Storage for assembly times
5029 {
5031 }
5032
5033#endif
5034
5035 //------------Assemble and populate the lists-----------------------
5036 {
5037 // Allocate local storage for the element's contribution to the
5038 // residuals vectors and system matrices of the size of the maximum
5039 // number of dofs in any element.
5040 // This means that the stored is only allocated (and deleted) once
5043
5044
5045 // Pointer to a single list to be used during the assembly
5046 std::list<std::pair<unsigned, double>>* list_pt;
5047
5048 // Loop over the all elements
5049 for (unsigned long e = el_lo; e <= el_hi; e++)
5050 {
5051#ifdef OOMPH_HAS_MPI
5052 // Time it?
5054 {
5056 }
5057#endif
5058
5059 // Get the pointer to the element
5061
5062#ifdef OOMPH_HAS_MPI
5063 // Ignore halo elements
5064 if (!elem_pt->is_halo())
5065 {
5066#endif
5067
5068 // Find number of degrees of freedom in the element
5069 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5070
5071 // Resize the storage for the elemental jacobian and residuals
5072 for (unsigned v = 0; v < n_vector; v++)
5073 {
5074 el_residuals[v].resize(nvar);
5075 }
5076 for (unsigned m = 0; m < n_matrix; m++)
5077 {
5078 el_jacobian[m].resize(nvar);
5079 }
5080
5081 // Now get the residuals and jacobian for the element
5084
5085 //---------------- Insert the values into the lists -----------
5086
5087 // Loop over the first index of local variables
5088 for (unsigned i = 0; i < nvar; i++)
5089 {
5090 // Get the local equation number
5091 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5092
5093 // Add the contribution to the residuals
5094 for (unsigned v = 0; v < n_vector; v++)
5095 {
5096 // Fill in the residuals vector
5097 residuals[v][eqn_number] += el_residuals[v][i];
5098 }
5099
5100 // Now loop over the other index
5101 for (unsigned j = 0; j < nvar; j++)
5102 {
5103 // Get the number of the unknown
5105
5106 // Loop over the matrices
5107 for (unsigned m = 0; m < n_matrix; m++)
5108 {
5109 // Get the value of the matrix at this point
5110 double value = el_jacobian[m](i, j);
5111 // Only add to theif it's non-zero
5112 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5113 {
5114 // If it's compressed row storage, then our vector is indexed
5115 // by row (the equation number)
5117 {
5118 // Find the list that corresponds to the desired row
5119 list_pt = &matrix_data_list[m][eqn_number];
5120 // Insert the data into the list, the first entry
5121 // in the pair is the unknown (column index),
5122 // the second is the value itself.
5123 list_pt->insert(list_pt->end(),
5124 std::make_pair(unknown, value));
5125 }
5126 // Otherwise it's compressed column storage, and our
5127 // vector is indexed by column (the unknown)
5128 else
5129 {
5130 // Find the list that corresponds to the desired column
5132 // Insert the data into the list, the first entry
5133 // in the pair is the equation number (row index),
5134 // the second is the value itself.
5135 list_pt->insert(list_pt->end(),
5136 std::make_pair(eqn_number, value));
5137 }
5138 }
5139 }
5140 }
5141 }
5142
5143#ifdef OOMPH_HAS_MPI
5144 } // endif halo element
5145#endif
5146
5147
5148#ifdef OOMPH_HAS_MPI
5149 // Time it?
5151 {
5154 }
5155#endif
5156
5157 } // End of loop over the elements
5158
5159 } // list_pt goes out of scope
5160
5161
5162#ifdef OOMPH_HAS_MPI
5163
5164 // Postprocess timing information and re-allocate distribution of
5165 // elements during subsequent assemblies.
5168 {
5170 }
5171
5172 // We have determined load balancing for current setup.
5173 // This can remain the same until assign_eqn_numbers() is called
5174 // again -- the flag is re-set to true there.
5176 {
5178 }
5179
5180#endif
5181
5182
5183 //----Finally we need to convert the beautiful list storage scheme---
5184 //----------to the containers required by SuperLU--------------------
5185
5186 // Loop over the number of matrices
5187 for (unsigned m = 0; m < n_matrix; m++)
5188 {
5189 // Set the number of rows or columns
5190 row_or_column_start[m] = new int[ndof + 1];
5191 // Counter for the total number of entries in the storage scheme
5192 unsigned long entry_count = 0;
5193 // The first entry is 0
5195
5196 // first we compute the number of non-zeros
5197 nnz[m] = 0;
5198 for (unsigned long i_global = 0; i_global < ndof; i_global++)
5199 {
5200 nnz[m] += matrix_data_list[m][i_global].size();
5201 }
5202
5203 // and then resize the storage
5204 column_or_row_index[m] = new int[nnz[m]];
5205 value[m] = new double[nnz[m]];
5206
5207 // Now we merely loop over the number of rows or columns
5208 for (unsigned long i_global = 0; i_global < ndof; i_global++)
5209 {
5210 // Start index for the present row is the number of entries so far
5212 // If there are no entries in the list then skip the loop
5214 {
5215 continue;
5216 }
5217
5218 // Sort the list corresponding to this row or column by the
5219 // column or row index (first entry in the pair).
5220 // This might be inefficient, but we only have to do the sort ONCE
5221 // for each list. This is faster than using a map storage scheme, where
5222 // we are sorting for every insertion (although the map structure
5223 // is cleaner and more memory efficient)
5224 matrix_data_list[m][i_global].sort();
5225
5226 // Set up an iterator for start of the list
5227 std::list<std::pair<unsigned, double>>::iterator it =
5228 matrix_data_list[m][i_global].begin();
5229
5230 // Get the first row or column index in the list...
5231 unsigned current_index = it->first;
5232 //...and the corresponding value
5233 double current_value = it->second;
5234
5235 // Loop over all the entries in the sorted list
5236 // Increase the iterator so that we start at the second entry
5237 for (++it; it != matrix_data_list[m][i_global].end(); ++it)
5238 {
5239 // If the index has not changed, then we must add the contribution
5240 // of the present entry to the value.
5241 // Additionally check that the entry is non-zero
5242 if ((it->first == current_index) &&
5243 (std::fabs(it->second) > Numerical_zero_for_sparse_assembly))
5244 {
5245 current_value += it->second;
5246 }
5247 // Otherwise, we have added all the contributions to the index
5248 // to current_value, so add it to the SuperLU data structure
5249 else
5250 {
5251 // Add the row or column index to the vector
5253 // Add the actual value to the vector
5254 value[m][entry_count] = current_value;
5255 // Increase the counter for the number of entries in each vector
5256 entry_count++;
5257
5258 // Set the index and value to be those of the current entry in the
5259 // list
5260 current_index = it->first;
5261 current_value = it->second;
5262 }
5263 } // End of loop over all list entries for this global row or column
5264
5265 // There are TWO special cases to consider.
5266 // If there is only one equation number in the list, then it
5267 // will NOT have been added. We test this case by comparing the
5268 // number of entries with those stored in row_or_column_start[i_global]
5269 // Otherwise
5270 // If the final entry in the list has the same index as the penultimate
5271 // entry, then it will NOT have been added to the SuperLU storage scheme
5272 // Check this by comparing the current_index with the final index
5273 // stored in the SuperLU scheme. If they are not the same, then
5274 // add the current_index and value.
5275
5276 // If single equation number in list
5277 if ((static_cast<int>(entry_count) == row_or_column_start[m][i_global])
5278 // If we have a single equation number, this will not be evaluated.
5279 // If we don't then we do the test to check that the final
5280 // entry is added
5281 || (static_cast<int>(current_index) !=
5283 {
5284 // Add the row or column index to the vector
5286 // Add the actual value to the vector
5287 value[m][entry_count] = current_value;
5288 // Increase the counter for the number of entries in each vector
5289 entry_count++;
5290 }
5291
5292 } // End of loop over the rows or columns of the entire matrix
5293
5294 // Final entry in the row/column start vector
5296 } // End of loop over matrices
5297
5299 {
5300 oomph_info << "Pausing at end of sparse assembly." << std::endl;
5301 pause("Check memory usage now.");
5302 }
5303 }
5304
5305
5306 //=====================================================================
5307 /// This is a (private) helper function that is used to assemble system
5308 /// matrices in compressed row or column format
5309 /// and compute residual vectors using vectors of pairs
5310 /// The default action is to assemble the jacobian matrix and
5311 /// residuals for the Newton method. The action can be
5312 /// overloaded at an elemental level by chaging the default
5313 /// behaviour of the function Element::get_all_vectors_and_matrices().
5314 /// column_or_row_index: Column [or row] index of given entry
5315 /// row_or_column_start: Index of first entry for given row [or column]
5316 /// value : Vector of nonzero entries
5317 /// residuals : Residual vector
5318 /// compressed_row_flag: Bool flag to indicate if storage format is
5319 /// compressed row [if false interpretation of
5320 /// arguments is as stated in square brackets].
5321 //=====================================================================
5325 Vector<double*>& value,
5326 Vector<unsigned>& nnz,
5329 {
5330 // Total number of elements
5331 const unsigned long n_elements = mesh_pt()->nelement();
5332
5333 // Default range of elements for distributed problems
5334 unsigned long el_lo = 0;
5335 unsigned long el_hi = n_elements - 1;
5336
5337#ifdef OOMPH_HAS_MPI
5338 // Otherwise just loop over a fraction of the elements
5339 // (This will either have been initialised in
5340 // Problem::set_default_first_and_last_element_for_assembly() or
5341 // will have been re-assigned during a previous assembly loop
5342 // Note that following the re-assignment only the entries
5343 // for the current processor are relevant.
5345 {
5348 }
5349#endif
5350
5351 // number of local eqns
5352 unsigned ndof = this->ndof();
5353
5354 // Find the number of vectors to be assembled
5355 const unsigned n_vector = residuals.size();
5356
5357 // Find the number of matrices to be assembled
5358 const unsigned n_matrix = column_or_row_index.size();
5359
5360 // Locally cache pointer to assembly handler
5362
5363#ifdef OOMPH_HAS_MPI
5364 bool doing_residuals = false;
5365 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5366 {
5367 doing_residuals = true;
5368 }
5369#endif
5370
5371// Error check dimensions
5372#ifdef PARANOID
5374 {
5375 std::ostringstream error_stream;
5376 error_stream << "Error: " << std::endl
5377 << "row_or_column_start.size() "
5378 << row_or_column_start.size() << " does not equal "
5379 << "column_or_row_index.size() "
5380 << column_or_row_index.size() << std::endl;
5381 throw OomphLibError(
5383 }
5384
5385 if (value.size() != n_matrix)
5386 {
5387 std::ostringstream error_stream;
5388 error_stream << "Error: " << std::endl
5389 << "value.size() " << value.size() << " does not equal "
5390 << "column_or_row_index.size() "
5391 << column_or_row_index.size() << std::endl
5392 << std::endl
5393 << std::endl;
5394 throw OomphLibError(
5396 }
5397#endif
5398
5399
5400 // The idea behind this sparse assembly routine is to use a Vector of
5401 // Vectors of pairs for each complete matrix.
5402 // Each inner Vector stores pairs and holds the row (or column) index
5403 // and the value of the matrix entry.
5404
5405 // Set up Vector of Vectors to store the entries of each matrix,
5406 // indexed by either the column or row.
5408
5409 // Loop over the number of matrices being assembled and resize
5410 // each Vector of Vectors to the number of rows or columns of the matrix
5411 for (unsigned m = 0; m < n_matrix; m++)
5412 {
5413 matrix_data[m].resize(ndof);
5414 }
5415
5416 // Resize the residuals vectors
5417 for (unsigned v = 0; v < n_vector; v++)
5418 {
5419 residuals[v] = new double[ndof];
5420 for (unsigned i = 0; i < ndof; i++)
5421 {
5422 residuals[v][i] = 0;
5423 }
5424 }
5425
5426#ifdef OOMPH_HAS_MPI
5427
5428 // Storage for assembly time for elements
5429 double t_assemble_start = 0.0;
5430
5431 // Storage for assembly times
5433 {
5435 }
5436
5437#endif
5438
5439 //----------------Assemble and populate the vector storage scheme--------
5440 {
5441 // Allocate local storage for the element's contribution to the
5442 // residuals vectors and system matrices of the size of the maximum
5443 // number of dofs in any element
5444 // This means that the storage is only allocated (and deleted) once
5447
5448 // Loop over the elements
5449 for (unsigned long e = el_lo; e <= el_hi; e++)
5450 {
5451#ifdef OOMPH_HAS_MPI
5452 // Time it?
5454 {
5456 }
5457#endif
5458
5459 // Get the pointer to the element
5461
5462#ifdef OOMPH_HAS_MPI
5463 // Ignore halo elements
5464 if (!elem_pt->is_halo())
5465 {
5466#endif
5467
5468 // Find number of degrees of freedom in the element
5469 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5470
5471 // Resize the storage for elemental jacobian and residuals
5472 for (unsigned v = 0; v < n_vector; v++)
5473 {
5474 el_residuals[v].resize(nvar);
5475 }
5476 for (unsigned m = 0; m < n_matrix; m++)
5477 {
5478 el_jacobian[m].resize(nvar);
5479 }
5480
5481 // Now get the residuals and jacobian for the element
5484
5485 //---------------Insert the values into the vectors--------------
5486
5487 // Loop over the first index of local variables
5488 for (unsigned i = 0; i < nvar; i++)
5489 {
5490 // Get the local equation number
5491 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5492
5493 // Add the contribution to the residuals
5494 for (unsigned v = 0; v < n_vector; v++)
5495 {
5496 // Fill in each residuals vector
5497 residuals[v][eqn_number] += el_residuals[v][i];
5498 }
5499
5500 // Now loop over the other index
5501 for (unsigned j = 0; j < nvar; j++)
5502 {
5503 // Get the number of the unknown
5505
5506 // Loop over the matrices
5507 // If it's compressed row storage, then our vector of maps
5508 // is indexed by row (equation number)
5509 for (unsigned m = 0; m < n_matrix; m++)
5510 {
5511 // Get the value of the matrix at this point
5512 double value = el_jacobian[m](i, j);
5513 // Only bother to add to the vector if it's non-zero
5514 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5515 {
5516 // If it's compressed row storage, then our vector of maps
5517 // is indexed by row (equation number)
5519 {
5520 // Find the correct position and add the data into the
5521 // vectors
5522 const unsigned size = matrix_data[m][eqn_number].size();
5523 for (unsigned k = 0; k <= size; k++)
5524 {
5525 if (k == size)
5526 {
5527 matrix_data[m][eqn_number].push_back(
5528 std::make_pair(unknown, value));
5529 break;
5530 }
5531 else if (matrix_data[m][eqn_number][k].first == unknown)
5532 {
5533 matrix_data[m][eqn_number][k].second += value;
5534 break;
5535 }
5536 }
5537 }
5538 // Otherwise it's compressed column storage and our vector is
5539 // indexed by column (the unknown)
5540 else
5541 {
5542 // Add the data into the vectors in the correct position
5543 const unsigned size = matrix_data[m][unknown].size();
5544 for (unsigned k = 0; k <= size; k++)
5545 {
5546 if (k == size)
5547 {
5548 matrix_data[m][unknown].push_back(
5549 std::make_pair(eqn_number, value));
5550 break;
5551 }
5552 else if (matrix_data[m][unknown][k].first == eqn_number)
5553 {
5554 matrix_data[m][unknown][k].second += value;
5555 break;
5556 }
5557 }
5558 }
5559 }
5560 } // End of loop over matrices
5561 }
5562 }
5563
5564#ifdef OOMPH_HAS_MPI
5565 } // endif halo element
5566#endif
5567
5568
5569#ifdef OOMPH_HAS_MPI
5570 // Time it?
5572 {
5575 }
5576#endif
5577
5578 } // End of loop over the elements
5579
5580
5581 } // End of vector assembly
5582
5583
5584#ifdef OOMPH_HAS_MPI
5585
5586 // Postprocess timing information and re-allocate distribution of
5587 // elements during subsequent assemblies.
5590 {
5592 }
5593
5594 // We have determined load balancing for current setup.
5595 // This can remain the same until assign_eqn_numbers() is called
5596 // again -- the flag is re-set to true there.
5598 {
5600 }
5601
5602#endif
5603
5604
5605 //-----------Finally we need to convert this vector storage scheme
5606 //------------------------to the containers required by SuperLU
5607
5608 // Loop over the number of matrices
5609 for (unsigned m = 0; m < n_matrix; m++)
5610 {
5611 // Set the number of rows or columns
5612 row_or_column_start[m] = new int[ndof + 1];
5613
5614 // fill row_or_column_start and find the number of entries
5615 row_or_column_start[m][0] = 0;
5616 for (unsigned long i = 0; i < ndof; i++)
5617 {
5618 row_or_column_start[m][i + 1] =
5620 }
5621 const unsigned entries = row_or_column_start[m][ndof];
5622
5623 // resize vectors
5624 column_or_row_index[m] = new int[entries];
5625 value[m] = new double[entries];
5626 nnz[m] = entries;
5627
5628 // Now we merely loop over the number of rows or columns
5629 for (unsigned long i_global = 0; i_global < ndof; i_global++)
5630 {
5631 // If there are no entries in the vector then skip the rest of the loop
5632 if (matrix_data[m][i_global].empty())
5633 {
5634 continue;
5635 }
5636
5637 // Loop over all the entries in the vectors corresponding to the given
5638 // row or column. It will NOT be ordered
5639 unsigned p = 0;
5640 for (int j = row_or_column_start[m][i_global];
5642 j++)
5643 {
5645 value[m][j] = matrix_data[m][i_global][p].second;
5646 ++p;
5647 }
5648 }
5649 } // End of the loop over the matrices
5650
5652 {
5653 oomph_info << "Pausing at end of sparse assembly." << std::endl;
5654 pause("Check memory usage now.");
5655 }
5656 }
5657
5658
5659 //=====================================================================
5660 /// This is a (private) helper function that is used to assemble system
5661 /// matrices in compressed row or column format
5662 /// and compute residual vectors using two vectors.
5663 /// The default action is to assemble the jacobian matrix and
5664 /// residuals for the Newton method. The action can be
5665 /// overloaded at an elemental level by chaging the default
5666 /// behaviour of the function Element::get_all_vectors_and_matrices().
5667 /// column_or_row_index: Column [or row] index of given entry
5668 /// row_or_column_start: Index of first entry for given row [or column]
5669 /// value : Vector of nonzero entries
5670 /// residuals : Residual vector
5671 /// compressed_row_flag: Bool flag to indicate if storage format is
5672 /// compressed row [if false interpretation of
5673 /// arguments is as stated in square brackets].
5674 //=====================================================================
5678 Vector<double*>& value,
5679 Vector<unsigned>& nnz,
5682 {
5683 // Total number of elements
5684 const unsigned long n_elements = mesh_pt()->nelement();
5685
5686 // Default range of elements for distributed problems
5687 unsigned long el_lo = 0;
5688 unsigned long el_hi = n_elements - 1;
5689
5690
5691#ifdef OOMPH_HAS_MPI
5692 // Otherwise just loop over a fraction of the elements
5693 // (This will either have been initialised in
5694 // Problem::set_default_first_and_last_element_for_assembly() or
5695 // will have been re-assigned during a previous assembly loop
5696 // Note that following the re-assignment only the entries
5697 // for the current processor are relevant.
5699 {
5702 }
5703#endif
5704
5705 // number of local eqns
5706 unsigned ndof = this->ndof();
5707
5708 // Find the number of vectors to be assembled
5709 const unsigned n_vector = residuals.size();
5710
5711 // Find the number of matrices to be assembled
5712 const unsigned n_matrix = column_or_row_index.size();
5713
5714 // Locally cache pointer to assembly handler
5716
5717#ifdef OOMPH_HAS_MPI
5718 bool doing_residuals = false;
5719 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
5720 {
5721 doing_residuals = true;
5722 }
5723#endif
5724
5725// Error check dimensions
5726#ifdef PARANOID
5728 {
5729 std::ostringstream error_stream;
5730 error_stream << "Error: " << std::endl
5731 << "row_or_column_start.size() "
5732 << row_or_column_start.size() << " does not equal "
5733 << "column_or_row_index.size() "
5734 << column_or_row_index.size() << std::endl;
5735 throw OomphLibError(
5737 }
5738
5739 if (value.size() != n_matrix)
5740 {
5741 std::ostringstream error_stream;
5742 error_stream << "Error: " << std::endl
5743 << "value.size() " << value.size() << " does not equal "
5744 << "column_or_row_index.size() "
5745 << column_or_row_index.size() << std::endl
5746 << std::endl
5747 << std::endl;
5748 throw OomphLibError(
5750 }
5751#endif
5752
5753 // The idea behind this sparse assembly routine is to use Vectors of
5754 // Vectors for the entries in each complete matrix. And a second
5755 // Vector of Vectors stores the global row (or column) indeces. This
5756 // will not have the memory overheads associated with the methods using
5757 // lists or maps, but insertion will be more costly.
5758
5759 // Set up two vector of vectors to store the entries of each matrix,
5760 // indexed by either the column or row. The entries of the vector for
5761 // each matrix correspond to all the rows or columns of that matrix.
5764
5765 // Loop over the number of matrices being assembled and resize
5766 // each vector of vectors to the number of rows or columns of the matrix
5767 for (unsigned m = 0; m < n_matrix; m++)
5768 {
5770 matrix_values[m].resize(ndof);
5771 }
5772
5773 // Resize the residuals vectors
5774 for (unsigned v = 0; v < n_vector; v++)
5775 {
5776 residuals[v] = new double[ndof];
5777 for (unsigned i = 0; i < ndof; i++)
5778 {
5779 residuals[v][i] = 0;
5780 }
5781 }
5782
5783#ifdef OOMPH_HAS_MPI
5784
5785 // Storage for assembly time for elements
5786 double t_assemble_start = 0.0;
5787
5788 // Storage for assembly times
5790 {
5792 }
5793
5794#endif
5795
5796
5797 //----------------Assemble and populate the vector storage scheme-------
5798 {
5799 // Allocate local storage for the element's contribution to the
5800 // residuals vectors and system matrices of the size of the maximum
5801 // number of dofs in any element
5802 // This means that the storage will only be allocated (and deleted) once
5805
5806 // Loop over the elements
5807 for (unsigned long e = el_lo; e <= el_hi; e++)
5808 {
5809#ifdef OOMPH_HAS_MPI
5810 // Time it?
5812 {
5814 }
5815#endif
5816
5817 // Get the pointer to the element
5819
5820#ifdef OOMPH_HAS_MPI
5821 // Ignore halo elements
5822 if (!elem_pt->is_halo())
5823 {
5824#endif
5825
5826 // Find number of degrees of freedom in the element
5827 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
5828
5829 // Resize the storage for elemental jacobian and residuals
5830 for (unsigned v = 0; v < n_vector; v++)
5831 {
5832 el_residuals[v].resize(nvar);
5833 }
5834 for (unsigned m = 0; m < n_matrix; m++)
5835 {
5836 el_jacobian[m].resize(nvar);
5837 }
5838
5839 // Now get the residuals and jacobian for the element
5842
5843 //---------------Insert the values into the vectors--------------
5844
5845 // Loop over the first index of local variables
5846 for (unsigned i = 0; i < nvar; i++)
5847 {
5848 // Get the local equation number
5849 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
5850
5851 // Add the contribution to the residuals
5852 for (unsigned v = 0; v < n_vector; v++)
5853 {
5854 // Fill in each residuals vector
5855 residuals[v][eqn_number] += el_residuals[v][i];
5856 }
5857
5858 // Now loop over the other index
5859 for (unsigned j = 0; j < nvar; j++)
5860 {
5861 // Get the number of the unknown
5863
5864 // Loop over the matrices
5865 // If it's compressed row storage, then our vector of maps
5866 // is indexed by row (equation number)
5867 for (unsigned m = 0; m < n_matrix; m++)
5868 {
5869 // Get the value of the matrix at this point
5870 double value = el_jacobian[m](i, j);
5871 // Only bother to add to the vector if it's non-zero
5872 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
5873 {
5874 // If it's compressed row storage, then our vector of maps
5875 // is indexed by row (equation number)
5877 {
5878 // Find the correct position and add the data into the
5879 // vectors
5880 const unsigned size =
5881 matrix_row_or_col_indices[m][eqn_number].size();
5882
5883 for (unsigned k = 0; k <= size; k++)
5884 {
5885 if (k == size)
5886 {
5887 matrix_row_or_col_indices[m][eqn_number].push_back(
5888 unknown);
5889 matrix_values[m][eqn_number].push_back(value);
5890 break;
5891 }
5892 else if (matrix_row_or_col_indices[m][eqn_number][k] ==
5893 unknown)
5894 {
5895 matrix_values[m][eqn_number][k] += value;
5896 break;
5897 }
5898 }
5899 }
5900 // Otherwise it's compressed column storage and our vector is
5901 // indexed by column (the unknown)
5902 else
5903 {
5904 // Add the data into the vectors in the correct position
5905 const unsigned size =
5907 for (unsigned k = 0; k <= size; k++)
5908 {
5909 if (k == size)
5910 {
5912 eqn_number);
5913 matrix_values[m][unknown].push_back(value);
5914 break;
5915 }
5916 else if (matrix_row_or_col_indices[m][unknown][k] ==
5917 eqn_number)
5918 {
5919 matrix_values[m][unknown][k] += value;
5920 break;
5921 }
5922 }
5923 }
5924 }
5925 } // End of loop over matrices
5926 }
5927 }
5928
5929#ifdef OOMPH_HAS_MPI
5930 } // endif halo element
5931#endif
5932
5933
5934#ifdef OOMPH_HAS_MPI
5935 // Time it?
5937 {
5940 }
5941#endif
5942
5943 } // End of loop over the elements
5944
5945 } // End of vector assembly
5946
5947
5948#ifdef OOMPH_HAS_MPI
5949
5950 // Postprocess timing information and re-allocate distribution of
5951 // elements during subsequent assemblies.
5954 {
5956 }
5957
5958 // We have determined load balancing for current setup.
5959 // This can remain the same until assign_eqn_numbers() is called
5960 // again -- the flag is re-set to true there.
5962 {
5964 }
5965
5966#endif
5967
5968 //-----------Finally we need to convert this lousy vector storage scheme
5969 //------------------------to the containers required by SuperLU
5970
5971 // Loop over the number of matrices
5972 for (unsigned m = 0; m < n_matrix; m++)
5973 {
5974 // Set the number of rows or columns
5975 row_or_column_start[m] = new int[ndof + 1];
5976
5977 // fill row_or_column_start and find the number of entries
5978 row_or_column_start[m][0] = 0;
5979 for (unsigned long i = 0; i < ndof; i++)
5980 {
5981 row_or_column_start[m][i + 1] =
5983 }
5984 const unsigned entries = row_or_column_start[m][ndof];
5985
5986 // resize vectors
5987 column_or_row_index[m] = new int[entries];
5988 value[m] = new double[entries];
5989 nnz[m] = entries;
5990
5991 // Now we merely loop over the number of rows or columns
5992 for (unsigned long i_global = 0; i_global < ndof; i_global++)
5993 {
5994 // If there are no entries in the vector then skip the rest of the loop
5995 if (matrix_values[m][i_global].empty())
5996 {
5997 continue;
5998 }
5999
6000 // Loop over all the entries in the vectors corresponding to the given
6001 // row or column. It will NOT be ordered
6002 unsigned p = 0;
6003 for (int j = row_or_column_start[m][i_global];
6005 j++)
6006 {
6008 value[m][j] = matrix_values[m][i_global][p];
6009 ++p;
6010 }
6011 }
6012 } // End of the loop over the matrices
6013
6015 {
6016 oomph_info << "Pausing at end of sparse assembly." << std::endl;
6017 pause("Check memory usage now.");
6018 }
6019 }
6020
6021
6022 //=====================================================================
6023 /// This is a (private) helper function that is used to assemble system
6024 /// matrices in compressed row or column format
6025 /// and compute residual vectors using two vectors.
6026 /// The default action is to assemble the jacobian matrix and
6027 /// residuals for the Newton method. The action can be
6028 /// overloaded at an elemental level by chaging the default
6029 /// behaviour of the function Element::get_all_vectors_and_matrices().
6030 /// column_or_row_index: Column [or row] index of given entry
6031 /// row_or_column_start: Index of first entry for given row [or column]
6032 /// value : Vector of nonzero entries
6033 /// residuals : Residual vector
6034 /// compressed_row_flag: Bool flag to indicate if storage format is
6035 /// compressed row [if false interpretation of
6036 /// arguments is as stated in square brackets].
6037 //=====================================================================
6041 Vector<double*>& value,
6042 Vector<unsigned>& nnz,
6045 {
6046 // Total number of elements
6047 const unsigned long n_elements = mesh_pt()->nelement();
6048
6049 // Default range of elements for distributed problems
6050 unsigned long el_lo = 0;
6051 unsigned long el_hi = n_elements - 1;
6052
6053
6054#ifdef OOMPH_HAS_MPI
6055 // Otherwise just loop over a fraction of the elements
6056 // (This will either have been initialised in
6057 // Problem::set_default_first_and_last_element_for_assembly() or
6058 // will have been re-assigned during a previous assembly loop
6059 // Note that following the re-assignment only the entries
6060 // for the current processor are relevant.
6062 {
6065 }
6066#endif
6067
6068 // number of local eqns
6069 unsigned ndof = this->ndof();
6070
6071 // Find the number of vectors to be assembled
6072 const unsigned n_vector = residuals.size();
6073
6074 // Find the number of matrices to be assembled
6075 const unsigned n_matrix = column_or_row_index.size();
6076
6077 // Locally cache pointer to assembly handler
6079
6080#ifdef OOMPH_HAS_MPI
6081 bool doing_residuals = false;
6082 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6083 {
6084 doing_residuals = true;
6085 }
6086#endif
6087
6088// Error check dimensions
6089#ifdef PARANOID
6091 {
6092 std::ostringstream error_stream;
6093 error_stream << "Error: " << std::endl
6094 << "row_or_column_start.size() "
6095 << row_or_column_start.size() << " does not equal "
6096 << "column_or_row_index.size() "
6097 << column_or_row_index.size() << std::endl;
6098 throw OomphLibError(
6100 }
6101
6102 if (value.size() != n_matrix)
6103 {
6104 std::ostringstream error_stream;
6105 error_stream << "Error: " << std::endl
6106 << "value.size() " << value.size() << " does not equal "
6107 << "column_or_row_index.size() "
6108 << column_or_row_index.size() << std::endl
6109 << std::endl
6110 << std::endl;
6111 throw OomphLibError(
6113 }
6114#endif
6115
6116 // The idea behind this sparse assembly routine is to use Vectors of
6117 // Vectors for the entries in each complete matrix. And a second
6118 // Vector of Vectors stores the global row (or column) indeces. This
6119 // will not have the memory overheads associated with the methods using
6120 // lists or maps, but insertion will be more costly.
6121
6122 // Set up two vector of vectors to store the entries of each matrix,
6123 // indexed by either the column or row. The entries of the vector for
6124 // each matrix correspond to all the rows or columns of that matrix.
6127
6128 // Loop over the number of matrices being assembled and resize
6129 // each vector of vectors to the number of rows or columns of the matrix
6130 for (unsigned m = 0; m < n_matrix; m++)
6131 {
6132 matrix_row_or_col_indices[m] = new unsigned*[ndof];
6133 matrix_values[m] = new double*[ndof];
6134 }
6135
6136 // Resize the residuals vectors
6137 for (unsigned v = 0; v < n_vector; v++)
6138 {
6139 residuals[v] = new double[ndof];
6140 for (unsigned i = 0; i < ndof; i++)
6141 {
6142 residuals[v][i] = 0;
6143 }
6144 }
6145
6146#ifdef OOMPH_HAS_MPI
6147
6148 // Storage for assembly time for elements
6149 double t_assemble_start = 0.0;
6150
6151 // Storage for assembly times
6153 {
6155 }
6156
6157#endif
6158
6159 // number of coefficients in each row
6161 for (unsigned m = 0; m < n_matrix; m++)
6162 {
6163 ncoef[m].resize(ndof, 0);
6164 }
6165
6167 {
6169 for (unsigned m = 0; m < n_matrix; m++)
6170 {
6172 }
6173 }
6174
6175 //----------------Assemble and populate the vector storage scheme-------
6176 {
6177 // Allocate local storage for the element's contribution to the
6178 // residuals vectors and system matrices of the size of the maximum
6179 // number of dofs in any element
6180 // This means that the storage will only be allocated (and deleted) once
6183
6184 // Loop over the elements
6185 for (unsigned long e = el_lo; e <= el_hi; e++)
6186 {
6187#ifdef OOMPH_HAS_MPI
6188 // Time it?
6190 {
6192 }
6193#endif
6194
6195 // Get the pointer to the element
6197
6198#ifdef OOMPH_HAS_MPI
6199 // Ignore halo elements
6200 if (!elem_pt->is_halo())
6201 {
6202#endif
6203
6204 // Find number of degrees of freedom in the element
6205 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6206
6207 // Resize the storage for elemental jacobian and residuals
6208 for (unsigned v = 0; v < n_vector; v++)
6209 {
6210 el_residuals[v].resize(nvar);
6211 }
6212 for (unsigned m = 0; m < n_matrix; m++)
6213 {
6214 el_jacobian[m].resize(nvar);
6215 }
6216
6217 // Now get the residuals and jacobian for the element
6220
6221 //---------------Insert the values into the vectors--------------
6222
6223 // Loop over the first index of local variables
6224 for (unsigned i = 0; i < nvar; i++)
6225 {
6226 // Get the local equation number
6227 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, i);
6228
6229 // Add the contribution to the residuals
6230 for (unsigned v = 0; v < n_vector; v++)
6231 {
6232 // Fill in each residuals vector
6233 residuals[v][eqn_number] += el_residuals[v][i];
6234 }
6235
6236 // Now loop over the other index
6237 for (unsigned j = 0; j < nvar; j++)
6238 {
6239 // Get the number of the unknown
6241
6242 // Loop over the matrices
6243 // If it's compressed row storage, then our vector of maps
6244 // is indexed by row (equation number)
6245 for (unsigned m = 0; m < n_matrix; m++)
6246 {
6247 // Get the value of the matrix at this point
6248 double value = el_jacobian[m](i, j);
6249 // Only bother to add to the vector if it's non-zero
6250 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6251 {
6252 // number of entrys in this row
6253 const unsigned size = ncoef[m][eqn_number];
6254
6255 // if no data has been allocated for this row then allocate
6256 if (size == 0)
6257 {
6258 // do we have previous allocation data
6260 [m][eqn_number] != 0)
6261 {
6262 matrix_row_or_col_indices[m][eqn_number] = new unsigned
6264 [m][eqn_number]];
6265 matrix_values[m][eqn_number] = new double
6267 [m][eqn_number]];
6268 }
6269 else
6270 {
6271 matrix_row_or_col_indices[m][eqn_number] = new unsigned
6273 matrix_values[m][eqn_number] = new double
6276 [m][eqn_number] =
6278 }
6279 }
6280
6281 // If it's compressed row storage, then our vector of maps
6282 // is indexed by row (equation number)
6284 {
6285 // next add the data
6286 for (unsigned k = 0; k <= size; k++)
6287 {
6288 if (k == size)
6289 {
6290 // do we need to allocate more storage
6292 [m][eqn_number] == ncoef[m][eqn_number])
6293 {
6294 unsigned new_allocation =
6295 ncoef[m][eqn_number] +
6297 double* new_values = new double[new_allocation];
6298 unsigned* new_indices = new unsigned[new_allocation];
6299 for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6300 {
6301 new_values[c] = matrix_values[m][eqn_number][c];
6302 new_indices[c] =
6303 matrix_row_or_col_indices[m][eqn_number][c];
6304 }
6305 delete[] matrix_values[m][eqn_number];
6306 delete[] matrix_row_or_col_indices[m][eqn_number];
6307 matrix_values[m][eqn_number] = new_values;
6308 matrix_row_or_col_indices[m][eqn_number] =
6311 [m][eqn_number] = new_allocation;
6312 }
6313 // and now add the data
6314 unsigned entry = ncoef[m][eqn_number];
6315 ncoef[m][eqn_number]++;
6316 matrix_row_or_col_indices[m][eqn_number][entry] =
6317 unknown;
6318 matrix_values[m][eqn_number][entry] = value;
6319 break;
6320 }
6321 else if (matrix_row_or_col_indices[m][eqn_number][k] ==
6322 unknown)
6323 {
6324 matrix_values[m][eqn_number][k] += value;
6325 break;
6326 }
6327 }
6328 }
6329 // Otherwise it's compressed column storage and our vector is
6330 // indexed by column (the unknown)
6331 else
6332 {
6333 // Add the data into the vectors in the correct position
6334 for (unsigned k = 0; k <= size; k++)
6335 {
6336 if (k == size)
6337 {
6338 // do we need to allocate more storage
6340 [m][unknown] == ncoef[m][unknown])
6341 {
6342 unsigned new_allocation =
6343 ncoef[m][unknown] +
6345 double* new_values = new double[new_allocation];
6346 unsigned* new_indices = new unsigned[new_allocation];
6347 for (unsigned c = 0; c < ncoef[m][unknown]; c++)
6348 {
6350 new_indices[c] =
6352 }
6353 delete[] matrix_values[m][unknown];
6357 }
6358 // and now add the data
6359 unsigned entry = ncoef[m][unknown];
6360 ncoef[m][unknown]++;
6362 eqn_number;
6363 matrix_values[m][unknown][entry] = value;
6364 break;
6365 }
6366 else if (matrix_row_or_col_indices[m][unknown][k] ==
6367 eqn_number)
6368 {
6369 matrix_values[m][unknown][k] += value;
6370 break;
6371 }
6372 }
6373 }
6374 }
6375 } // End of loop over matrices
6376 }
6377 }
6378
6379#ifdef OOMPH_HAS_MPI
6380 } // endif halo element
6381#endif
6382
6383
6384#ifdef OOMPH_HAS_MPI
6385 // Time it?
6387 {
6390 }
6391#endif
6392
6393 } // End of loop over the elements
6394
6395 } // End of vector assembly
6396
6397
6398#ifdef OOMPH_HAS_MPI
6399
6400 // Postprocess timing information and re-allocate distribution of
6401 // elements during subsequent assemblies.
6404 {
6406 }
6407
6408 // We have determined load balancing for current setup.
6409 // This can remain the same until assign_eqn_numbers() is called
6410 // again -- the flag is re-set to true there.
6412 {
6414 }
6415
6416#endif
6417
6418 //-----------Finally we need to convert this lousy vector storage scheme
6419 //------------------------to the containers required by SuperLU
6420
6421 // Loop over the number of matrices
6422 for (unsigned m = 0; m < n_matrix; m++)
6423 {
6424 // Set the number of rows or columns
6425 row_or_column_start[m] = new int[ndof + 1];
6426
6427 // fill row_or_column_start and find the number of entries
6428 row_or_column_start[m][0] = 0;
6429 for (unsigned long i = 0; i < ndof; i++)
6430 {
6433 }
6434 const unsigned entries = row_or_column_start[m][ndof];
6435
6436 // resize vectors
6437 column_or_row_index[m] = new int[entries];
6438 value[m] = new double[entries];
6439 nnz[m] = entries;
6440
6441 // Now we merely loop over the number of rows or columns
6442 for (unsigned long i_global = 0; i_global < ndof; i_global++)
6443 {
6444 // If there are no entries in the vector then skip the rest of the loop
6445 if (ncoef[m][i_global] == 0)
6446 {
6447 continue;
6448 }
6449
6450 // Loop over all the entries in the vectors corresponding to the given
6451 // row or column. It will NOT be ordered
6452 unsigned p = 0;
6453 for (int j = row_or_column_start[m][i_global];
6455 j++)
6456 {
6458 value[m][j] = matrix_values[m][i_global][p];
6459 ++p;
6460 }
6461
6462 // and delete
6464 delete[] matrix_values[m][i_global];
6465 }
6466
6467 //
6468 delete[] matrix_row_or_col_indices[m];
6469 delete[] matrix_values[m];
6470 } // End of the loop over the matrices
6471
6473 {
6474 oomph_info << "Pausing at end of sparse assembly." << std::endl;
6475 pause("Check memory usage now.");
6476 }
6477 }
6478
6479
6480#ifdef OOMPH_HAS_MPI
6481 //=======================================================================
6482 /// Helper method that returns the global equations to which
6483 /// the elements in the range el_lo to el_hi contribute on this
6484 /// processor
6485 //=======================================================================
6486 void Problem::get_my_eqns(AssemblyHandler* const& assembly_handler_pt,
6487 const unsigned& el_lo,
6488 const unsigned& el_hi,
6490 {
6491 // Index to keep track of the equations counted
6492 unsigned my_eqns_index = 0;
6493
6494 // Loop over the selection of elements
6495 for (unsigned long e = el_lo; e <= el_hi; e++)
6496 {
6497 // Get the pointer to the element
6499
6500 // Ignore halo elements
6501 if (!elem_pt->is_halo())
6502 {
6503 // Find number of degrees of freedom in the element
6504 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6505 // Add the number of dofs to the current size of my_eqns
6506 my_eqns.resize(my_eqns_index + nvar);
6507
6508 // Loop over the first index of local variables
6509 for (unsigned i = 0; i < nvar; i++)
6510 {
6511 // Get the local equation number
6512 unsigned global_eqn_number =
6514 // Add into the vector
6515 my_eqns[my_eqns_index + i] = global_eqn_number;
6516 }
6517 // Update the number of elements in the vector
6519 }
6520 }
6521
6522 // now sort and remove duplicate entries in the vector
6523 std::sort(my_eqns.begin(), my_eqns.end());
6524 Vector<unsigned>::iterator it = std::unique(my_eqns.begin(), my_eqns.end());
6525 my_eqns.resize(it - my_eqns.begin());
6526 }
6527
6528
6529 //=============================================================================
6530 /// Helper method to assemble CRDoubleMatrices from distributed
6531 /// on multiple processors.
6532 //=============================================================================
6536 Vector<int*>& row_start,
6537 Vector<double*>& values,
6538 Vector<unsigned>& nnz,
6540 {
6541 // Time assembly
6542 double t_start = TimingHelpers::timer();
6543
6544 // my rank and nproc
6545 unsigned my_rank = Communicator_pt->my_rank();
6546 unsigned nproc = Communicator_pt->nproc();
6547
6548 // Total number of elements
6549 const unsigned long n_elements = mesh_pt()->nelement();
6550
6551#ifdef PARANOID
6552 // No elements? This is usually a sign that the problem distribution has
6553 // led to one processor not having any elements. Either
6554 // a sign of something having gone wrong or a relatively small
6555 // problem on a huge number of processors
6556 if (n_elements == 0)
6557 {
6558 std::ostringstream error_stream;
6559 error_stream << "Processsor " << my_rank << " has no elements. \n"
6560 << "This is usually a sign that the problem distribution \n"
6561 << "or the load balancing have gone wrong.";
6563 "Problem::parallel_sparse_assemble()",
6565 }
6566#endif
6567
6568
6569 // Default range of elements for distributed problems.
6570 unsigned long el_lo = 0;
6571 unsigned long el_hi_plus_one = n_elements;
6572
6573 // Otherwise just loop over a fraction of the elements
6574 // (This will either have been initialised in
6575 // Problem::set_default_first_and_last_element_for_assembly() or
6576 // will have been re-assigned during a previous assembly loop
6577 // Note that following the re-assignment only the entries
6578 // for the current processor are relevant.
6580 {
6583 }
6584
6585 // Find the number of vectors to be assembled
6586 const unsigned n_vector = residuals.size();
6587
6588 // Find the number of matrices to be assembled
6589 const unsigned n_matrix = column_indices.size();
6590
6591 // Locally cache pointer to assembly handler
6593
6594 bool doing_residuals = false;
6595 if (dynamic_cast<ParallelResidualsHandler*>(Assembly_handler_pt) != 0)
6596 {
6597 doing_residuals = true;
6598 }
6599
6600// Error check dimensions
6601#ifdef PARANOID
6602 if (row_start.size() != n_matrix)
6603 {
6604 std::ostringstream error_stream;
6605 error_stream << "Error: " << std::endl
6606 << "row_or_column_start.size() " << row_start.size()
6607 << " does not equal "
6608 << "column_or_row_index.size() " << column_indices.size()
6609 << std::endl;
6610 throw OomphLibError(
6612 }
6613
6614 if (values.size() != n_matrix)
6615 {
6616 std::ostringstream error_stream;
6617 error_stream << "Error: " << std::endl
6618 << "value.size() " << values.size() << " does not equal "
6619 << "column_or_row_index.size() " << column_indices.size()
6620 << std::endl
6621 << std::endl
6622 << std::endl;
6623 throw OomphLibError(
6625 }
6626#endif
6627
6628
6629 // start by assembling the sorted set of equations to which this processor
6630 // contributes. Essentially this is every global equation that features in
6631 // all the non-halo elements. This may not be the same as the locally-stored
6632 // dofs because some of the Nodes in non-halo elements may actually
6633 // be halos.
6634 //======================================================================
6636 if (n_elements != 0)
6637 {
6638 this->get_my_eqns(
6639 assembly_handler_pt, el_lo, el_hi_plus_one - 1, my_eqns);
6640 }
6641
6642 // number of equations
6643 unsigned my_n_eqn = my_eqns.size();
6644
6645 // next we assemble the data into an array of arrays
6646 // =================================================
6647 // The idea behind this sparse assembly routine is to use an array of
6648 // arrays for the entries in each complete matrix. And a second
6649 // array of arrays stores the global row (or column) indeces.
6650
6651 // Set up two vector of vectors to store the entries of each matrix,
6652 // indexed by either the column or row. The entries of the vector for
6653 // each matrix correspond to all the rows or columns of that matrix.
6656
6657 // Loop over the number of matrices being assembled and resize
6658 // each vector of vectors to the number of rows or columns of the matrix
6659 for (unsigned m = 0; m < n_matrix; m++)
6660 {
6661 matrix_col_indices[m] = new unsigned*[my_n_eqn];
6662 matrix_values[m] = new double*[my_n_eqn];
6663 for (unsigned i = 0; i < my_n_eqn; i++)
6664 {
6665 matrix_col_indices[m][i] = 0;
6666 matrix_values[m][i] = 0;
6667 }
6668 }
6669
6670 // Resize the residuals vectors
6672 for (unsigned v = 0; v < n_vector; v++)
6673 {
6674 residuals_data[v] = new double[my_n_eqn];
6675 for (unsigned i = 0; i < my_n_eqn; i++)
6676 {
6677 residuals_data[v][i] = 0;
6678 }
6679 }
6680
6681 // Storage for assembly time for elements
6682 double t_assemble_start = 0.0;
6683
6684 // Storage for assembly times
6686 {
6688 }
6689
6690 // number of coefficients in each row
6692 for (unsigned m = 0; m < n_matrix; m++)
6693 {
6694 ncoef[m].resize(my_n_eqn, 0);
6695 }
6696
6697 // Sparse_assemble_with_arrays_previous_allocation stores the number of
6698 // coefs in each row.
6699 // if a matrix of this size has not been assembled before then resize this
6700 // storage
6702 {
6704 for (unsigned m = 0; m < n_matrix; m++)
6705 {
6707 }
6708 }
6709
6710
6711 // assemble and populate an array based storage scheme
6712 {
6713 // Allocate local storage for the element's contribution to the
6714 // residuals vectors and system matrices of the size of the maximum
6715 // number of dofs in any element
6716 // This means that the storage will only be allocated (and deleted) once
6719
6720 // Loop over the elements
6721 for (unsigned long e = el_lo; e < el_hi_plus_one; e++)
6722 {
6723 // Time it?
6725 {
6727 }
6728
6729 // Get the pointer to the element
6731
6732 // Ignore halo elements
6733 if (!elem_pt->is_halo())
6734 {
6735 // Find number of degrees of freedom in the element
6736 const unsigned nvar = assembly_handler_pt->ndof(elem_pt);
6737
6738 // Resize the storage for elemental jacobian and residuals
6739 for (unsigned v = 0; v < n_vector; v++)
6740 {
6741 el_residuals[v].resize(nvar);
6742 }
6743 for (unsigned m = 0; m < n_matrix; m++)
6744 {
6745 el_jacobian[m].resize(nvar);
6746 }
6747
6748 // Now get the residuals and jacobian for the element
6751
6752 //---------------Insert the values into the vectors--------------
6753
6754 // Loop over the first index of local variables
6755 for (unsigned i = 0; i < nvar; i++)
6756 {
6757 // Get the local equation number
6758 unsigned global_eqn_number =
6760
6761 // determine the element number in my set of eqns using the
6762 // bisection method
6763 int left = 0;
6764 int right = my_n_eqn - 1;
6765 int eqn_number = right / 2;
6766 while (my_eqns[eqn_number] != global_eqn_number)
6767 {
6768 if (left == right)
6769 {
6770 // Check that the residuals associated with the
6771 // eqn number that can't be found are all zero
6772 bool broken = false;
6773 for (unsigned v = 0; v < n_vector; v++)
6774 {
6775 if (el_residuals[v][i] != 0.0)
6776 {
6777 broken = true;
6778 break;
6779 }
6780 }
6781
6782 // Now loop over the other index to check the entries
6783 // in the appropriate row of the Jacobians are zero too
6784 for (unsigned j = 0; j < nvar; j++)
6785 {
6786 // Get the number of the unknown
6787 // unsigned unknown =
6788 // assembly_handler_pt->eqn_number(elem_pt,j);
6789
6790 // Loop over the matrices
6791 // If it's compressed row storage, then our vector of maps
6792 // is indexed by row (equation number)
6793 for (unsigned m = 0; m < n_matrix; m++)
6794 {
6795 // Get the value of the matrix at this point
6796 double value = el_jacobian[m](i, j);
6797 if (value != 0.0)
6798 {
6799 broken = true;
6800 break;
6801 }
6802 if (broken) break;
6803 }
6804 }
6805
6806 if (broken)
6807 {
6808 std::ostringstream error_stream;
6810 << "Internal Error: " << std::endl
6811 << "Could not find global equation number "
6812 << global_eqn_number
6813 << " in my_eqns vector of equation numbers but\n"
6814 << "at least one entry in the residual vector is nonzero.";
6815 throw OomphLibError(error_stream.str(),
6818 }
6819 else
6820 {
6821 break;
6822 }
6823 }
6824 if (my_eqns[eqn_number] > global_eqn_number)
6825 {
6826 right = std::max(eqn_number - 1, left);
6827 }
6828 else
6829 {
6830 left = std::min(eqn_number + 1, right);
6831 }
6832 eqn_number = (right + left) / 2;
6833 }
6834
6835 // Add the contribution to the residuals
6836 for (unsigned v = 0; v < n_vector; v++)
6837 {
6838 // Fill in each residuals vector
6839 residuals_data[v][eqn_number] += el_residuals[v][i];
6840 }
6841
6842 // Now loop over the other index
6843 for (unsigned j = 0; j < nvar; j++)
6844 {
6845 // Get the number of the unknown
6847
6848 // Loop over the matrices
6849 // If it's compressed row storage, then our vector of maps
6850 // is indexed by row (equation number)
6851 for (unsigned m = 0; m < n_matrix; m++)
6852 {
6853 // Get the value of the matrix at this point
6854 double value = el_jacobian[m](i, j);
6855 // Only bother to add to the vector if it's non-zero
6856 if (std::fabs(value) > Numerical_zero_for_sparse_assembly)
6857 {
6858 // number of entrys in this row
6859 const unsigned size = ncoef[m][eqn_number];
6860
6861 // if no data has been allocated for this row then allocate
6862 if (size == 0)
6863 {
6864 // do we have previous allocation data
6866 [m][eqn_number] != 0)
6867 {
6868 matrix_col_indices[m][eqn_number] = new unsigned
6870 [m][eqn_number]];
6871
6872 matrix_values[m][eqn_number] = new double
6874 [m][eqn_number]];
6875 }
6876 else
6877 {
6878 matrix_col_indices[m][eqn_number] = new unsigned
6880
6881 matrix_values[m][eqn_number] = new double
6883
6885 [m][eqn_number] =
6887 }
6888 }
6889
6890 // next add the data
6891 for (unsigned k = 0; k <= size; k++)
6892 {
6893 if (k == size)
6894 {
6895 // do we need to allocate more storage
6897 [m][eqn_number] == ncoef[m][eqn_number])
6898 {
6899 unsigned new_allocation =
6900 ncoef[m][eqn_number] +
6902 double* new_values = new double[new_allocation];
6903 unsigned* new_indices = new unsigned[new_allocation];
6904 for (unsigned c = 0; c < ncoef[m][eqn_number]; c++)
6905 {
6906 new_values[c] = matrix_values[m][eqn_number][c];
6907 new_indices[c] = matrix_col_indices[m][eqn_number][c];
6908 }
6909 delete[] matrix_values[m][eqn_number];
6910 delete[] matrix_col_indices[m][eqn_number];
6911
6912 matrix_values[m][eqn_number] = new_values;
6913 matrix_col_indices[m][eqn_number] = new_indices;
6914
6916 [m][eqn_number] = new_allocation;
6917 }
6918 // and now add the data
6919 unsigned entry = ncoef[m][eqn_number];
6920 ncoef[m][eqn_number]++;
6921 matrix_col_indices[m][eqn_number][entry] = unknown;
6922 matrix_values[m][eqn_number][entry] = value;
6923 break;
6924 }
6925 else if (matrix_col_indices[m][eqn_number][k] == unknown)
6926 {
6927 matrix_values[m][eqn_number][k] += value;
6928 break;
6929 }
6930 }
6931 } // numerical zero check
6932 } // End of loop over matrices
6933 }
6934 }
6935 } // endif halo element
6936
6937 // Time it?
6939 {
6942 }
6943 } // End of loop over the elements
6944 } // End of vector assembly
6945
6946
6947 // Doc?
6948 double t_end = 0.0;
6949 double t_local = 0.0;
6950 double t_max = 0.0;
6951 double t_min = 0.0;
6952 double t_sum = 0.0;
6954 {
6956 t_local = t_end - t_start;
6957 t_max = 0.0;
6958 t_min = 0.0;
6959 t_sum = 0.0;
6961 &t_max,
6962 1,
6963 MPI_DOUBLE,
6964 MPI_MAX,
6965 this->communicator_pt()->mpi_comm());
6967 &t_min,
6968 1,
6969 MPI_DOUBLE,
6970 MPI_MIN,
6971 this->communicator_pt()->mpi_comm());
6973 &t_sum,
6974 1,
6975 MPI_DOUBLE,
6976 MPI_SUM,
6977 this->communicator_pt()->mpi_comm());
6978 double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
6979
6980 if (doing_residuals)
6981 {
6982 oomph_info << "\nCPU for residual computation (loc/max/min/imbal): ";
6983 }
6984 else
6985 {
6986 oomph_info << "\nCPU for Jacobian computation (loc/max/min/imbal): ";
6987 }
6988 oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
6989 << "%\n";
6990
6992 }
6993
6994
6995 // Adjust number of coefficients in each row
6996 for (unsigned m = 0; m < n_matrix; m++)
6997 {
6998 unsigned max = 0;
6999 unsigned min = INT_MAX;
7000 unsigned sum = 0;
7001 unsigned sum_total = 0;
7002 for (unsigned e = 0; e < my_n_eqn; e++)
7003 {
7004 sum += ncoef[m][e];
7006 if (ncoef[m][e] > max) max = ncoef[m][e];
7007 if (ncoef[m][e] < min) min = ncoef[m][e];
7008
7009 // Now shrink the storage to what we actually need
7010 unsigned new_allocation = ncoef[m][e];
7011 double* new_values = new double[new_allocation];
7012 unsigned* new_indices = new unsigned[new_allocation];
7013 for (unsigned c = 0; c < ncoef[m][e]; c++)
7014 {
7015 new_values[c] = matrix_values[m][e][c];
7017 }
7018 delete[] matrix_values[m][e];
7019 delete[] matrix_col_indices[m][e];
7020
7023 }
7024 }
7025
7026
7027 // Postprocess timing information and re-allocate distribution of
7028 // elements during subsequent assemblies.
7031 {
7033 }
7034
7035 // We have determined load balancing for current setup.
7036 // This can remain the same until assign_eqn_numbers() is called
7037 // again -- the flag is re-set to true there.
7039 {
7041 }
7042
7043
7044 // next we compute the number of equations and number of non-zeros to be
7045 // sent to each processor, and send/recv that information
7046 // =====================================================================
7047
7048 // determine the number of eqns to be sent to each processor
7051 // If no equations are assembled then we don't need to do any of this
7052 if (my_n_eqn > 0)
7053 {
7054 unsigned current_p = target_dist_pt->rank_of_global_row(my_eqns[0]);
7057 for (unsigned i = 1; i < my_n_eqn; i++)
7058 {
7059 unsigned next_p = target_dist_pt->rank_of_global_row(my_eqns[i]);
7060 if (next_p != current_p)
7061 {
7062 current_p = next_p;
7064 }
7066 }
7067 }
7068
7069 // determine the number of non-zeros to be sent to each processor for each
7070 // matrix (if n_eqn_for_proc[p]=0, then nothing will be assembled)
7072 for (unsigned p = 0; p < nproc; p++)
7073 {
7076 for (unsigned m = 0; m < n_matrix; m++)
7077 {
7078 for (int i = first_eqn_element; i <= last_eqn_element; i++)
7079 {
7080 nnz_for_proc(p, m) += ncoef[m][i];
7081 }
7082 }
7083 }
7084
7085 // next post the sends and recvs to the corresponding processors
7090 for (unsigned p = 0; p < nproc; p++)
7091 {
7092 if (p != my_rank)
7093 {
7094 temp_send_storage[p] = new unsigned[n_matrix + 1];
7096 for (unsigned m = 0; m < n_matrix; m++)
7097 {
7098 temp_send_storage[p][m + 1] = nnz_for_proc(p, m);
7099 }
7102 n_matrix + 1,
7104 p,
7105 0,
7106 Communicator_pt->mpi_comm(),
7107 &sreq);
7108 send_nnz_reqs.push_back(sreq);
7109 temp_recv_storage[p] = new unsigned[n_matrix + 1];
7112 n_matrix + 1,
7114 p,
7115 0,
7116 Communicator_pt->mpi_comm(),
7117 &rreq);
7118 recv_nnz_reqs.push_back(rreq);
7119 }
7120 }
7121
7122 // assemble the data to be sent to each processor
7123 // ==============================================
7124
7125 // storage
7131
7132 // equation numbers
7133 for (unsigned p = 0; p < nproc; p++)
7134 {
7135 unsigned n_eqns_p = n_eqn_for_proc[p];
7136 if (n_eqns_p > 0)
7137 {
7139 unsigned first_row = target_dist_pt->first_row(p);
7140 eqns_for_proc[p] = new unsigned[n_eqns_p];
7141 for (unsigned i = 0; i < n_eqns_p; i++)
7142 {
7143 eqns_for_proc[p][i] = my_eqns[i + first_eqn_element] - first_row;
7144 }
7145 }
7146 }
7147
7148 // residuals for p
7149 for (unsigned v = 0; v < n_vector; v++)
7150 {
7151 for (unsigned p = 0; p < nproc; p++)
7152 {
7153 unsigned n_eqns_p = n_eqn_for_proc[p];
7154 if (n_eqns_p > 0)
7155 {
7157 residuals_for_proc(p, v) = new double[n_eqns_p];
7158 for (unsigned i = 0; i < n_eqns_p; i++)
7159 {
7160 residuals_for_proc(p, v)[i] =
7162 }
7163 }
7164 }
7165 delete[] residuals_data[v];
7166 }
7167
7168 // matrices for p
7169 for (unsigned m = 0; m < n_matrix; m++)
7170 {
7171 for (unsigned p = 0; p < nproc; p++)
7172 {
7173 unsigned n_eqns_p = n_eqn_for_proc[p];
7174 if (n_eqns_p > 0)
7175 {
7177 row_start_for_proc(p, m) = new unsigned[n_eqns_p + 1];
7178 column_indices_for_proc(p, m) = new unsigned[nnz_for_proc(p, m)];
7179 values_for_proc(p, m) = new double[nnz_for_proc(p, m)];
7180 unsigned entry = 0;
7181 for (unsigned i = 0; i < n_eqns_p; i++)
7182 {
7183 row_start_for_proc(p, m)[i] = entry;
7184 unsigned n_coef_in_row = ncoef[m][first_eqn_element + i];
7185 for (unsigned j = 0; j < n_coef_in_row; j++)
7186 {
7187 column_indices_for_proc(p, m)[entry] =
7189 values_for_proc(p, m)[entry] =
7191 entry++;
7192 }
7193 }
7194 row_start_for_proc(p, m)[n_eqns_p] = entry;
7195 }
7196 }
7197 for (unsigned i = 0; i < my_n_eqn; i++)
7198 {
7199 delete[] matrix_col_indices[m][i];
7200 delete[] matrix_values[m][i];
7201 }
7202 delete[] matrix_col_indices[m];
7203 delete[] matrix_values[m];
7204 }
7205
7206 // need to wait for the recv nnzs to complete
7207 // before we can allocate storage for the matrix recvs
7208 // ===================================================
7209
7210 // recv and copy the datafrom the recv storage to
7211 // + nnz_from_proc
7212 // + n_eqn_from_proc
7217 for (unsigned p = 0; p < nproc; p++)
7218 {
7219 if (p != my_rank)
7220 {
7222 for (unsigned m = 0; m < n_matrix; m++)
7223 {
7225 }
7226 delete[] temp_recv_storage[p];
7227 }
7228 else
7229 {
7231 for (unsigned m = 0; m < n_matrix; m++)
7232 {
7234 }
7235 }
7236 }
7237 recv_nnz_stat.clear();
7238 recv_nnz_reqs.clear();
7239
7240 // allocate the storage for the data to be recv and post the sends recvs
7241 // =====================================================================
7242
7243 // storage
7249
7250 // allocate and post sends and recvs
7251 double base;
7254 unsigned n_comm_types = 1 + 1 * n_vector + 3 * n_matrix;
7257 for (unsigned p = 0; p < nproc; p++)
7258 {
7259 if (p != my_rank)
7260 {
7261 // allocate
7262 if (n_eqn_from_proc[p] > 0)
7263 {
7264 eqns_from_proc[p] = new unsigned[n_eqn_from_proc[p]];
7265 for (unsigned v = 0; v < n_vector; v++)
7266 {
7267 residuals_from_proc(p, v) = new double[n_eqn_from_proc[p]];
7268 }
7269 for (unsigned m = 0; m < n_matrix; m++)
7270 {
7271 row_start_from_proc(p, m) = new unsigned[n_eqn_from_proc[p] + 1];
7272 column_indices_from_proc(p, m) = new unsigned[nnz_from_proc(p, m)];
7273 values_from_proc(p, m) = new double[nnz_from_proc(p, m)];
7274 }
7275 }
7276
7277 // recv
7278 if (n_eqn_from_proc[p] > 0)
7279 {
7282 int count[n_comm_types];
7283 int pt = 0;
7284
7285 // equations
7286 count[pt] = 1;
7291 pt++;
7292
7293 // vectors
7294 for (unsigned v = 0; v < n_vector; v++)
7295 {
7296 count[pt] = 1;
7301 pt++;
7302 }
7303
7304 // matrices
7305 for (unsigned m = 0; m < n_matrix; m++)
7306 {
7307 // row start
7308 count[pt] = 1;
7314 pt++;
7315
7316
7317 // column indices
7318 count[pt] = 1;
7323 pt++;
7324
7325 // values
7326 count[pt] = 1;
7331 pt++;
7332 }
7333
7334 // build the combined type
7339 for (unsigned t = 0; t < n_comm_types; t++)
7340 {
7342 }
7344 MPI_Irecv(
7345 &base, 1, recv_type, p, 1, Communicator_pt->mpi_comm(), &req);
7347 recv_reqs.push_back(req);
7348 }
7349
7350 // send
7351 if (n_eqn_for_proc[p] > 0)
7352 {
7355 int count[n_comm_types];
7356 int pt = 0;
7357
7358 // equations
7359 count[pt] = 1;
7364 pt++;
7365
7366 // vectors
7367 for (unsigned v = 0; v < n_vector; v++)
7368 {
7369 count[pt] = 1;
7374 pt++;
7375 }
7376
7377 // matrices
7378 for (unsigned m = 0; m < n_matrix; m++)
7379 {
7380 // row start
7381 count[pt] = 1;
7387 pt++;
7388
7389
7390 // column indices
7391 count[pt] = 1;
7396 pt++;
7397
7398 // values
7399 count[pt] = 1;
7404 pt++;
7405 }
7406
7407 // build the combined type
7412 for (unsigned t = 0; t < n_comm_types; t++)
7413 {
7415 }
7417 MPI_Isend(
7418 &base, 1, send_type, p, 1, Communicator_pt->mpi_comm(), &req);
7420 send_reqs.push_back(req);
7421 }
7422 }
7423 // otherwise send to self
7424 else
7425 {
7427 for (unsigned v = 0; v < n_vector; v++)
7428 {
7430 }
7431 for (unsigned m = 0; m < n_matrix; m++)
7432 {
7436 }
7437 }
7438 }
7439
7440 // wait for the recvs to complete
7441 unsigned n_recv_req = recv_reqs.size();
7442 if (n_recv_req > 0)
7443 {
7446 }
7447
7448 // ==============================================
7449 unsigned target_nrow_local = target_dist_pt->nrow_local();
7450
7451 // loop over the matrices
7452 for (unsigned m = 0; m < n_matrix; m++)
7453 {
7454 // allocate row_start
7455 row_start[m] = new int[target_nrow_local + 1];
7456 row_start[m][0] = 0;
7457
7458 // initially allocate storage based on the maximum number of non-zeros
7459 // from any one processor
7461 for (unsigned p = 0; p < nproc; p++)
7462 {
7464 }
7466 values_chunk[0] = new double[nnz_allocation];
7472 unsigned current_chunk = 0;
7473
7474 // for each row on this processor
7475 for (unsigned i = 0; i < target_nrow_local; i++)
7476 {
7477 row_start[m][i] = 0;
7478
7479 // determine the processors that this row is on
7481 for (unsigned p = 0; p < nproc; p++)
7482 {
7483 if (n_eqn_from_proc[p] == 0)
7484 {
7485 row_on_proc[p] = -1;
7486 }
7487 else
7488 {
7489 int left = 0;
7490 int right = n_eqn_from_proc[p] - 1;
7491 int midpoint = right / 2;
7492 bool complete = false;
7493 while (!complete)
7494 {
7495 midpoint = (right + left) / 2;
7496 if (midpoint > right)
7497 {
7498 midpoint = right;
7499 }
7500 if (midpoint < left)
7501 {
7502 midpoint = left;
7503 }
7504 if (left == right)
7505 {
7506 if (eqns_from_proc[p][midpoint] == i)
7507 {
7508 midpoint = left;
7509 }
7510 else
7511 {
7512 midpoint = -1;
7513 }
7514 complete = true;
7515 }
7516 else if (eqns_from_proc[p][midpoint] == i)
7517 {
7518 complete = true;
7519 }
7520 else if (eqns_from_proc[p][midpoint] > i)
7521 {
7522 right = std::max(midpoint - 1, left);
7523 }
7524 else
7525 {
7526 left = std::min(midpoint + 1, right);
7527 }
7528 }
7530 }
7531 }
7532
7533 // for each processor build this row of the matrix
7535 unsigned check_last = check_first;
7536 for (unsigned p = 0; p < nproc; p++)
7537 {
7538 if (row_on_proc[p] != -1)
7539 {
7540 int row = row_on_proc[p];
7541 unsigned first = row_start_from_proc(p, m)[row];
7542 unsigned last = row_start_from_proc(p, m)[row + 1];
7543 for (unsigned l = first; l < last; l++)
7544 {
7545 bool done = false;
7546 for (unsigned j = check_first; j <= check_last && !done; j++)
7547 {
7548 if (j == check_last)
7549 {
7550 // is this temp array full, do we need to allocate
7551 // a new temp array
7554 {
7555 // number of chunks allocated
7556 unsigned n_chunk = values_chunk.size();
7557
7558 // determine the number of non-zeros added so far
7559 // (excluding the current row)
7560 unsigned nnz_so_far = 0;
7561 for (unsigned c = 0; c < n_chunk; c++)
7562 {
7564 }
7565 nnz_so_far -= row_start[m][i];
7566
7567 // average number of non-zeros per row
7568 unsigned avg_nnz = nnz_so_far / (i + 1);
7569
7570 // number of rows left +1
7571 unsigned nrows_left = target_nrow_local - i;
7572
7573 // allocation for next chunk
7574 unsigned next_chunk_size =
7575 avg_nnz * nrows_left + row_start[m][i];
7576
7577 // allocate storage in next chunk
7578 current_chunk++;
7579 n_chunk++;
7580 values_chunk.resize(n_chunk);
7584 new int[next_chunk_size];
7585 size_of_chunk.resize(n_chunk);
7587 ncoef_in_chunk.resize(n_chunk);
7588
7589 // copy current row from previous chunk to new chunk
7590 for (unsigned k = check_first; k < check_last; k++)
7591 {
7596 }
7597 ncoef_in_chunk[current_chunk - 1] -= row_start[m][i];
7598 ncoef_in_chunk[current_chunk] = row_start[m][i];
7599
7600 // update first_check and last_check
7601 check_first = 0;
7602 check_last = row_start[m][i];
7603 j = check_last;
7604 }
7605
7606 // add the coefficient
7611 row_start[m][i]++;
7612 check_last++;
7613 done = true;
7614 }
7616 (int)column_indices_from_proc(p, m)[l])
7617 {
7619 done = true;
7620 }
7621 }
7622 }
7623 }
7624 }
7625 }
7626
7627 // delete recv data for this matrix
7628 for (unsigned p = 0; p < nproc; p++)
7629 {
7630 if (n_eqn_from_proc[p] > 0)
7631 {
7632 delete[] row_start_from_proc(p, m);
7633 delete[] column_indices_from_proc(p, m);
7634 delete[] values_from_proc(p, m);
7635 }
7636 }
7637
7638 // next we take the chunk base storage of the column indices and values
7639 // and copy into a single contiguous block of memory
7640 // ====================================================================
7641 unsigned n_chunk = values_chunk.size();
7642 nnz[m] = 0;
7643 for (unsigned c = 0; c < n_chunk; c++)
7644 {
7645 nnz[m] += ncoef_in_chunk[c];
7646 }
7648
7649 // allocate
7650 values[m] = new double[nnz[m]];
7651 column_indices[m] = new int[nnz[m]];
7652
7653 // copy
7654 unsigned pt = 0;
7655 for (unsigned c = 0; c < n_chunk; c++)
7656 {
7657 unsigned nc = ncoef_in_chunk[c];
7658 for (unsigned i = 0; i < nc; i++)
7659 {
7660 values[m][pt + i] = values_chunk[c][i];
7662 }
7663 pt += nc;
7664 delete[] values_chunk[c];
7665 delete[] column_indices_chunk[c];
7666 }
7667
7668 // the row_start vector currently contains the number of coefs in each
7669 // row. Update
7670 // ===================================================================
7671 unsigned g = row_start[m][0];
7672 row_start[m][0] = 0;
7673 for (unsigned i = 1; i < target_nrow_local; i++)
7674 {
7675 unsigned h = g + row_start[m][i];
7676 row_start[m][i] = g;
7677 g = h;
7678 }
7679 row_start[m][target_nrow_local] = g;
7680 }
7681
7682 // next accumulate the residuals
7683 for (unsigned v = 0; v < n_vector; v++)
7684 {
7685 residuals[v] = new double[target_nrow_local];
7686 for (unsigned i = 0; i < target_nrow_local; i++)
7687 {
7688 residuals[v][i] = 0;
7689 }
7690 for (unsigned p = 0; p < nproc; p++)
7691 {
7692 if (n_eqn_from_proc[p] > 0)
7693 {
7694 unsigned n_eqn_p = n_eqn_from_proc[p];
7695 for (unsigned i = 0; i < n_eqn_p; i++)
7696 {
7698 }
7699 delete[] residuals_from_proc(p, v);
7700 }
7701 }
7702 }
7703
7704 // delete list of eqns from proc
7705 for (unsigned p = 0; p < nproc; p++)
7706 {
7707 if (n_eqn_from_proc[p] > 0)
7708 {
7709 delete[] eqns_from_proc[p];
7710 }
7711 }
7712
7713 // and wait for sends to complete
7716 for (unsigned p = 0; p < nproc; p++)
7717 {
7718 if (p != my_rank)
7719 {
7720 delete[] temp_send_storage[p];
7721 }
7722 }
7723 send_nnz_stat.clear();
7724 send_nnz_reqs.clear();
7725
7726 // wait for the matrix data sends to complete and delete the data
7727 unsigned n_send_reqs = send_reqs.size();
7728 if (n_send_reqs > 0)
7729 {
7732 for (unsigned p = 0; p < nproc; p++)
7733 {
7734 if (p != my_rank)
7735 {
7736 if (n_eqn_for_proc[p])
7737 {
7738 delete[] eqns_for_proc[p];
7739 for (unsigned m = 0; m < n_matrix; m++)
7740 {
7741 delete[] row_start_for_proc(p, m);
7742 delete[] column_indices_for_proc(p, m);
7743 delete[] values_for_proc(p, m);
7744 }
7745 for (unsigned v = 0; v < n_vector; v++)
7746 {
7747 delete[] residuals_for_proc(p, v);
7748 }
7749 }
7750 }
7751 }
7752 }
7753
7754 // Doc?
7756 {
7758 t_local = t_end - t_start;
7759 t_max = 0.0;
7760 t_min = 0.0;
7761 t_sum = 0.0;
7763 &t_max,
7764 1,
7765 MPI_DOUBLE,
7766 MPI_MAX,
7767 this->communicator_pt()->mpi_comm());
7769 &t_min,
7770 1,
7771 MPI_DOUBLE,
7772 MPI_MIN,
7773 this->communicator_pt()->mpi_comm());
7775 &t_sum,
7776 1,
7777 MPI_DOUBLE,
7778 MPI_SUM,
7779 this->communicator_pt()->mpi_comm());
7780 double imbalance = (t_max - t_min) / (t_sum / double(nproc)) * 100.0;
7781 if (doing_residuals)
7782 {
7783 oomph_info << "CPU for residual distribut. (loc/max/min/imbal): ";
7784 }
7785 else
7786 {
7787 oomph_info << "CPU for Jacobian distribut. (loc/max/min/imbal): ";
7788 }
7789 oomph_info << t_local << " " << t_max << " " << t_min << " " << imbalance
7790 << "%\n\n";
7791 }
7792 }
7793
7794#endif
7795
7796
7797 //================================================================
7798 /// Get the full Jacobian by finite differencing
7799 //================================================================
7801 DenseMatrix<double>& jacobian)
7802 {
7803#ifdef OOMPH_HAS_MPI
7804
7806 {
7807 OomphLibWarning("This is unlikely to work with a distributed problem",
7808 " Problem::get_fd_jacobian()",
7810 }
7811#endif
7812
7813
7814 // Find number of dofs
7815 const unsigned long n_dof = ndof();
7816
7817 // Advanced residuals
7819
7820 // Get reference residuals
7822
7823 const double FD_step = 1.0e-8;
7824
7825 // Make sure the Jacobian is the right size (since we don't care about
7826 // speed).
7827 jacobian.resize(n_dof, n_dof);
7828
7829 // Loop over all dofs
7830 for (unsigned long jdof = 0; jdof < n_dof; jdof++)
7831 {
7832 double backup = *Dof_pt[jdof];
7833 *Dof_pt[jdof] += FD_step;
7834
7835 // We're checking if the new values for Dof_pt[] actually
7836 // solve the entire problem --> update as if problem had
7837 // been solved
7841
7842 // Get advanced residuals
7844
7845 for (unsigned long ieqn = 0; ieqn < n_dof; ieqn++)
7846 {
7847 jacobian(ieqn, jdof) =
7848 (residuals_pls[ieqn] - residuals[ieqn]) / FD_step;
7849 }
7850
7851 *Dof_pt[jdof] = backup;
7852 }
7853
7854 // Reset problem to state it was in
7858 }
7859
7860 //======================================================================
7861 /// Get derivative of the residuals vector wrt a global parameter
7862 /// This is required in continuation problems
7863 //=======================================================================
7866 {
7867 // If we are doing the calculation analytically then call the appropriate
7868 // handler and then calling get_residuals
7870 {
7871 // Locally cache pointer to assembly handler
7873 // Create a new assembly handler that replaces get_residuals by
7874 // get_dresiduals_dparameter for each element
7877 // Get the residuals, which will be dresiduals by dparameter
7878 this->get_residuals(result);
7879 // Delete the parameter derivative handler
7880 delete Assembly_handler_pt;
7881 // Reset the assembly handler to the original handler
7883
7884 /*AssemblyHandler* const assembly_handler_pt = Assembly_handler_pt;
7885 //Loop over all the elements
7886 unsigned long Element_pt_range = Mesh_pt->nelement();
7887 for(unsigned long e=0;e<Element_pt_range;e++)
7888 {
7889 //Get the pointer to the element
7890 GeneralisedElement* elem_pt = Mesh_pt->element_pt(e);
7891 //Find number of dofs in the element
7892 unsigned n_element_dofs = assembly_handler_pt->ndof(elem_pt);
7893 //Set up an array
7894 Vector<double> element_residuals(n_element_dofs);
7895 //Fill the array
7896 assembly_handler_pt->get_dresiduals_dparameter(elem_pt,parameter_pt,
7897 element_residuals);
7898 //Now loop over the dofs and assign values to global Vector
7899 for(unsigned l=0;l<n_element_dofs;l++)
7900 {
7901 result[assembly_handler_pt->eqn_number(elem_pt,l)]
7902 += element_residuals[l];
7903 }
7904 }*/
7905
7906 // for(unsigned n=0;n<n_dof;n++)
7907 // {std::cout << "BLA " << n << " " << result[n] << "\n";}
7908 }
7909 // Otherwise use the finite difference default
7910 else
7911 {
7912 // Get the (global) residuals and store in the result vector
7914
7915 // Storage for the new residuals
7917
7918 // Increase the global parameter
7919 const double FD_step = 1.0e-8;
7920
7921 // Store the current value of the parameter
7922 double param_value = *parameter_pt;
7923
7924 // Increase the parameter
7925 *parameter_pt += FD_step;
7926
7927 // Do any possible updates
7929
7930 // Get the new residuals
7932
7933 // Find the number of local rows
7934 //(I think it's a global vector, so that should be fine)
7935 const unsigned ndof_local = result.nrow_local();
7936
7937 // Do the finite differencing in the local variables
7938 for (unsigned n = 0; n < ndof_local; ++n)
7939 {
7940 result[n] = (newres[n] - result[n]) / FD_step;
7941 }
7942
7943 // Reset the value of the parameter
7945
7946 // Do any possible updates
7948 }
7949 }
7950
7951
7952 //======================================================================
7953 /// Return the product of the global hessian (derivative of Jacobian
7954 /// matrix with respect to all variables) with
7955 /// an eigenvector, Y, and any number of other specified vectors C
7956 /// (d(J_{ij})/d u_{k}) Y_{j} C_{k}.
7957 /// This function is used in assembling and solving the augmented systems
7958 /// associated with bifurcation tracking.
7959 /// The default implementation is to use finite differences at the global
7960 /// level.
7961 //========================================================================
7966 {
7967 // How many vector products must we construct
7968 const unsigned n_vec = C.size();
7969
7970 // currently only global (non-distributed) distributions are allowed
7971 // LinearAlgebraDistribution* dist_pt = new
7972 // LinearAlgebraDistribution(Communicator_pt,n_dof,false);
7973
7974 // Cache the assembly hander
7976
7977 // Rebuild the results vectors and initialise to zero
7978 // use the same distribution of the vector Y
7979 for (unsigned i = 0; i < n_vec; i++)
7980 {
7981 product[i].build(Y.distribution_pt(), 0.0);
7982 product[i].initialise(0.0);
7983 }
7984
7985// Setup the halo schemes for the result
7986#ifdef OOMPH_HAS_MPI
7988 {
7989 for (unsigned i = 0; i < n_vec; i++)
7990 {
7991 product[i].build_halo_scheme(this->Halo_scheme_pt);
7992 }
7993 }
7994#endif
7995
7996 // If we are doing the calculation analytically then call the appropriate
7997 // handler
7998 // A better way to do this is probably to hook into the get_residuals
7999 // framework but with a different member function of the assembly
8000 // handler
8002 {
8003 // Loop over all the elements
8004 unsigned long Element_pt_range = Mesh_pt->nelement();
8005 for (unsigned long e = 0; e < Element_pt_range; e++)
8006 {
8007 // Get the pointer to the element
8009// Do not loop over halo elements
8010#ifdef OOMPH_HAS_MPI
8011 if (!elem_pt->is_halo())
8012 {
8013#endif
8014 // Find number of dofs in the element
8016 // Set up a matrix for the input and output
8020
8021 // Translate the global input vectors into the local storage
8022 // Probably horribly inefficient, but otherwise things get really
8023 // messy at the elemental level
8024 for (unsigned l = 0; l < n_var; l++)
8025 {
8026 // Cache the global equation number
8027 const unsigned long eqn_number =
8029
8030 Y_local[l] = Y.global_value(eqn_number);
8031 for (unsigned i = 0; i < n_vec; i++)
8032 {
8033 C_local(i, l) = C[i].global_value(eqn_number);
8034 }
8035 }
8036
8037 // Fill the array
8040
8041 // Assign the local results to the global vector
8042 for (unsigned l = 0; l < n_var; l++)
8043 {
8044 const unsigned long eqn_number =
8046
8047 for (unsigned i = 0; i < n_vec; i++)
8048 {
8049 product[i].global_value(eqn_number) += product_local(i, l);
8050 // std::cout << "BLA " << e << " " << i << " "
8051 // << l << " " << product_local(i,l) << "\n";
8052 }
8053 }
8054#ifdef OOMPH_HAS_MPI
8055 }
8056#endif
8057 }
8058 }
8059 // Otherwise calculate using finite differences by
8060 // perturbing the jacobian along a particular direction
8061 else
8062 {
8063 // Cache the finite difference step
8064 /// Alice: My bifurcation tracking converges better with this FD_step
8065 /// as 1.0e-5. The default value remains at 1.0e-8.
8066 const double FD_step = FD_step_used_in_get_hessian_vector_products;
8067
8068 // We can now construct our multipliers
8069 const unsigned n_dof_local = this->Dof_distribution_pt->nrow_local();
8070 // Prepare to scale
8071 double dof_length = 0.0;
8073
8074 for (unsigned n = 0; n < n_dof_local; n++)
8075 {
8076 if (std::fabs(this->dof(n)) > dof_length)
8077 {
8078 dof_length = std::fabs(this->dof(n));
8079 }
8080 }
8081
8082 // C is assumed to have the same distribution as the dofs
8083 for (unsigned i = 0; i < n_vec; i++)
8084 {
8085 for (unsigned n = 0; n < n_dof_local; n++)
8086 {
8087 if (std::fabs(C[i][n]) > C_length[i])
8088 {
8089 C_length[i] = std::fabs(C[i][n]);
8090 }
8091 }
8092 }
8093
8094 // Now broadcast the information, if distributed
8095#ifdef OOMPH_HAS_MPI
8097 {
8098 const unsigned n_length = n_vec + 1;
8099 double all_length[n_length];
8101 for (unsigned i = 0; i < n_vec; i++)
8102 {
8103 all_length[i + 1] = C_length[i];
8104 }
8105
8106 // Do the MPI call
8110 n_length,
8111 MPI_DOUBLE,
8112 MPI_MAX,
8113 this->communicator_pt()->mpi_comm());
8114
8115 // Read out the information
8117 for (unsigned i = 0; i < n_vec; i++)
8118 {
8119 C_length[i] = all_length_reduce[i + 1];
8120 }
8121 }
8122#endif
8123
8124 // Form the multipliers
8126 for (unsigned i = 0; i < n_vec; i++)
8127 {
8129 C_mult[i] += FD_step;
8130 C_mult[i] *= FD_step;
8131 }
8132
8133
8134 // Dummy vector to stand in the place of the residuals
8136
8137 // Calculate the product of the jacobian matrices, etc by looping over the
8138 // elements
8139 const unsigned long n_element = this->mesh_pt()->nelement();
8140 for (unsigned long e = 0; e < n_element; e++)
8141 {
8143 // Ignore halo's of course
8144#ifdef OOMPH_HAS_MPI
8145 if (!elem_pt->is_halo())
8146 {
8147#endif
8148 // Loop over the ndofs in each element
8150 // Resize the dummy residuals vector
8151 dummy_res.resize(n_var);
8152 // Allocate storage for the unperturbed jacobian matrix
8154 // Get unperturbed jacobian
8156
8157 // Backup the dofs
8159 for (unsigned n = 0; n < n_var; n++)
8160 {
8161 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8162 dof_bac[n] = *this->global_dof_pt(eqn_number);
8163 }
8164
8165 // Now loop over all vectors C
8166 for (unsigned i = 0; i < n_vec; i++)
8167 {
8168 // Perturb the dofs by the appropriate vector
8169 for (unsigned n = 0; n < n_var; n++)
8170 {
8171 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8172 // Perturb by vector C[i]
8173 *this->global_dof_pt(eqn_number) +=
8174 C_mult[i] * C[i].global_value(eqn_number);
8175 }
8177
8178 // Allocate storage for the perturbed jacobian
8180
8181 // Now get the new jacobian
8183
8184 // Reset the dofs
8185 for (unsigned n = 0; n < n_var; n++)
8186 {
8187 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8188 *this->global_dof_pt(eqn_number) = dof_bac[n];
8189 }
8191
8192 // Now work out the products
8193 for (unsigned n = 0; n < n_var; n++)
8194 {
8195 unsigned eqn_number = assembly_handler_pt->eqn_number(elem_pt, n);
8196 double prod_c = 0.0;
8197 for (unsigned m = 0; m < n_var; m++)
8198 {
8200 prod_c += (jac_C(n, m) - jac(n, m)) * Y.global_value(unknown);
8201 }
8202 // std::cout << "FD " << e << " " << i << " "
8203 // << n << " " << prod_c/C_mult[i] << "\n";
8204 product[i].global_value(eqn_number) += prod_c / C_mult[i];
8205 }
8206 }
8207#ifdef OOMPH_HAS_MPI
8208 }
8209#endif
8210 } // End of loop over elements
8211 }
8212
8213 // If we have a distributed problem then gather all
8214 // values
8215#ifdef OOMPH_HAS_MPI
8217 {
8218 // Sum all values if distributed
8219 for (unsigned i = 0; i < n_vec; i++)
8220 {
8221 product[i].sum_all_halo_and_haloed_values();
8222 }
8223 }
8224#endif
8225 }
8226
8227
8228 //================================================================
8229 /// Get derivative of an element in the problem wrt a global
8230 /// parameter, to be used in continuation problems
8231 //================================================================
8232 /*void Problem::get_derivative_wrt_global_parameter(
8233 double* const &parameter_pt,
8234 GeneralisedElement* const &elem_pt,
8235 Vector<double> &result)
8236 {
8237
8238 #ifdef OOMPH_HAS_MPI
8239
8240 if (Problem_has_been_distributed)
8241 {
8242 OomphLibWarning("This is unlikely to work with a distributed problem",
8243 "Problem::get_derivative_wrt_global_parameter()",
8244 OOMPH_EXCEPTION_LOCATION);
8245 }
8246 #endif
8247
8248 //Locally cache pointer to assembly handler
8249 AssemblyHandler* const assembly_handler_pt = Assembly_handler_pt;
8250
8251 //Should definitely give this a more global scope
8252 double FD_Jstep = 1.0e-8;
8253
8254 //Find the number of variables in the element, e
8255 unsigned nvar = assembly_handler_pt->ndof(elem_pt);
8256 //Create storage for residuals
8257 Vector<double> residuals(nvar), newres(nvar);
8258
8259 //Get the "original" residuals
8260 assembly_handler_pt->get_residuals(elem_pt,residuals);
8261
8262 //Save the old value of the global parameter
8263 double old_var = *parameter_pt;
8264
8265 //Increment the value
8266 *parameter_pt += FD_Jstep;
8267
8268 //Now do any possible updates
8269 actions_after_change_in_global_parameter();
8270
8271 //Get the "new" residuals
8272 assembly_handler_pt->get_residuals(elem_pt,newres);
8273
8274 //Do the finite differences
8275 for(unsigned m=0;m<nvar;m++)
8276 {
8277 result[m] = (newres[m] - residuals[m])/FD_Jstep;
8278 }
8279
8280 //Reset value of the global parameter
8281 *parameter_pt = old_var;
8282
8283 //Now do any possible updates
8284 actions_after_change_in_global_parameter();
8285 }*/
8286
8287 //==================================================================
8288 /// Solve the eigenproblem. Legacy version that returns real vectors which are
8289 /// related in some solver-specific way to the real and imaginary parts
8290 /// of the actual, usually complex eigenvalues.
8291 /// At least n_eval eigenvalues are computed.
8292 //==================================================================
8294 const unsigned& n_eval,
8295 Vector<std::complex<double>>& eigenvalue,
8297 const bool& make_timesteppers_steady)
8298 {
8299 // If the boolean flag is steady, then make all the timesteppers steady
8300 // before solving the eigenproblem. This will "switch off" the
8301 // time-derivative terms in the jacobian matrix
8303 {
8304 // Find out how many timesteppers there are
8305 const unsigned n_time_steppers = ntime_stepper();
8306
8307 // Vector of bools to store the is_steady status of the various
8308 // timesteppers when we came in here
8309 std::vector<bool> was_steady(n_time_steppers);
8310
8311 // Loop over them all and make them (temporarily) static
8312 for (unsigned i = 0; i < n_time_steppers; i++)
8313 {
8316 }
8317
8318 const bool do_adjoint_problem = false;
8319 // Call the Eigenproblem for the eigensolver
8321 this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8322
8323 // Reset the is_steady status of all timesteppers that
8324 // weren't already steady when we came in here and reset their
8325 // weights
8326 for (unsigned i = 0; i < n_time_steppers; i++)
8327 {
8328 if (!was_steady[i])
8329 {
8331 }
8332 }
8333 }
8334 // Otherwise if we don't want to make the problem steady, just
8335 // assemble and solve the eigensystem
8336 else
8337 {
8338 const bool do_adjoint_problem = false;
8339 // Call the Eigenproblem for the eigensolver
8341 this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8342 }
8343 }
8344
8345 //==================================================================
8346 /// Solve the eigenproblem
8347 //==================================================================
8349 Vector<std::complex<double>>& alpha,
8350 Vector<double>& beta,
8353 const bool& make_timesteppers_steady)
8354 {
8355 // If the boolean flag is steady, then make all the timesteppers steady
8356 // before solving the eigenproblem. This will "switch off" the
8357 // time-derivative terms in the jacobian matrix
8359 {
8360 // Find out how many timesteppers there are
8361 const unsigned n_time_steppers = ntime_stepper();
8362
8363 // Vector of bools to store the is_steady status of the various
8364 // timesteppers when we came in here
8365 std::vector<bool> was_steady(n_time_steppers);
8366
8367 // Loop over them all and make them (temporarily) static
8368 for (unsigned i = 0; i < n_time_steppers; i++)
8369 {
8372 }
8373
8374 const bool do_adjoint_problem = false;
8375 // Call the Eigenproblem for the eigensolver
8377 n_eval,
8378 alpha,
8379 beta,
8383
8384 // Reset the is_steady status of all timesteppers that
8385 // weren't already steady when we came in here and reset their
8386 // weights
8387 for (unsigned i = 0; i < n_time_steppers; i++)
8388 {
8389 if (!was_steady[i])
8390 {
8392 }
8393 }
8394 }
8395 // Otherwise if we don't want to make the problem steady, just
8396 // assemble and solve the eigensystem
8397 else
8398 {
8399 const bool do_adjoint_problem = false;
8400 // Call the Eigenproblem for the eigensolver
8402 n_eval,
8403 alpha,
8404 beta,
8408 }
8409 }
8410
8411
8412 //==================================================================
8413 /// Solve the eigenproblem
8414 //==================================================================
8416 Vector<std::complex<double>>& eigenvalue,
8419 const bool& make_timesteppers_steady)
8420 {
8421 // If the boolean flag is steady, then make all the timesteppers steady
8422 // before solving the eigenproblem. This will "switch off" the
8423 // time-derivative terms in the jacobian matrix
8425 {
8426 // Find out how many timesteppers there are
8427 const unsigned n_time_steppers = ntime_stepper();
8428
8429 // Vector of bools to store the is_steady status of the various
8430 // timesteppers when we came in here
8431 std::vector<bool> was_steady(n_time_steppers);
8432
8433 // Loop over them all and make them (temporarily) static
8434 for (unsigned i = 0; i < n_time_steppers; i++)
8435 {
8438 }
8439
8440 const bool do_adjoint_problem = false;
8441 // Call the Eigenproblem for the eigensolver
8443 n_eval,
8444 eigenvalue,
8448
8449 // Reset the is_steady status of all timesteppers that
8450 // weren't already steady when we came in here and reset their
8451 // weights
8452 for (unsigned i = 0; i < n_time_steppers; i++)
8453 {
8454 if (!was_steady[i])
8455 {
8457 }
8458 }
8459 }
8460 // Otherwise if we don't want to make the problem steady, just
8461 // assemble and solve the eigensystem
8462 else
8463 {
8464 const bool do_adjoint_problem = false;
8465 // Call the Eigenproblem for the eigensolver
8467 n_eval,
8468 eigenvalue,
8472 }
8473 }
8474
8475
8476 //==================================================================
8477 /// Solve the adjoint eigenproblem
8478 //==================================================================
8480 const unsigned& n_eval,
8481 Vector<std::complex<double>>& eigenvalue,
8483 const bool& make_timesteppers_steady)
8484 {
8485 // If the boolean flag is steady, then make all the timesteppers steady
8486 // before solving the eigenproblem. This will "switch off" the
8487 // time-derivative terms in the jacobian matrix
8489 {
8490 // Find out how many timesteppers there are
8491 const unsigned n_time_steppers = ntime_stepper();
8492
8493 // Vector of bools to store the is_steady status of the various
8494 // timesteppers when we came in here
8495 std::vector<bool> was_steady(n_time_steppers);
8496
8497 // Loop over them all and make them (temporarily) static
8498 for (unsigned i = 0; i < n_time_steppers; i++)
8499 {
8502 }
8503
8504 const bool do_adjoint_problem = true;
8505 // Call the Eigenproblem for the ajoint-problem eigensolver
8506 // NB Only different to solve_eigenproblem
8508 this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8509
8510 // Reset the is_steady status of all timesteppers that
8511 // weren't already steady when we came in here and reset their
8512 // weights
8513 for (unsigned i = 0; i < n_time_steppers; i++)
8514 {
8515 if (!was_steady[i])
8516 {
8518 }
8519 }
8520 }
8521 // Otherwise if we don't want to make the problem steady, just
8522 // assemble and solve the eigensystem
8523 else
8524 {
8525 const bool do_adjoint_problem = true;
8526 // Call the Eigenproblem for the eigensolver
8528 this, n_eval, eigenvalue, eigenvector, do_adjoint_problem);
8529 }
8530 }
8531
8532
8533 //==================================================================
8534 /// Solve the adjoint eigenproblem
8535 //==================================================================
8537 const unsigned& n_eval,
8538 Vector<std::complex<double>>& eigenvalue,
8541 const bool& make_timesteppers_steady)
8542 {
8543 // If the boolean flag is steady, then make all the timesteppers steady
8544 // before solving the eigenproblem. This will "switch off" the
8545 // time-derivative terms in the jacobian matrix
8547 {
8548 // Find out how many timesteppers there are
8549 const unsigned n_time_steppers = ntime_stepper();
8550
8551 // Vector of bools to store the is_steady status of the various
8552 // timesteppers when we came in here
8553 std::vector<bool> was_steady(n_time_steppers);
8554
8555 // Loop over them all and make them (temporarily) static
8556 for (unsigned i = 0; i < n_time_steppers; i++)
8557 {
8560 }
8561
8562 const bool do_adjoint_problem = true;
8563 // Call the Eigenproblem for the eigensolver
8565 n_eval,
8566 eigenvalue,
8570
8571 // Reset the is_steady status of all timesteppers that
8572 // weren't already steady when we came in here and reset their
8573 // weights
8574 for (unsigned i = 0; i < n_time_steppers; i++)
8575 {
8576 if (!was_steady[i])
8577 {
8579 }
8580 }
8581 }
8582 // Otherwise if we don't want to make the problem steady, just
8583 // assemble and solve the eigensystem
8584 else
8585 {
8586 const bool do_adjoint_problem = true;
8587 // Call the Eigenproblem for the eigensolver
8589 n_eval,
8590 eigenvalue,
8594 }
8595 }
8596
8597 //===================================================================
8598 /// Get the matrices required to solve an eigenproblem
8599 /// WARNING: temporarily this method only works with non-distributed
8600 /// matrices
8601 //===================================================================
8604 const double& shift)
8605 {
8606 // Three different cases again here:
8607 // 1) Compiled with MPI, but run in serial
8608 // 2) Compiled with MPI, but MPI not initialised in driver
8609 // 3) Serial version
8610
8611
8612#ifdef PARANOID
8613 if (mass_matrix.distribution_built() && main_matrix.distribution_built())
8614 {
8615 // Check that the distribution of the mass matrix and jacobian match
8616 if (!(*mass_matrix.distribution_pt() == *main_matrix.distribution_pt()))
8617 {
8618 std::ostringstream error_stream;
8620 << "The distributions of the jacobian and mass matrix are\n"
8621 << "not the same and they must be.\n";
8622 throw OomphLibError(
8624 }
8625
8626 if (mass_matrix.nrow() != this->ndof())
8627 {
8628 std::ostringstream error_stream;
8630 << "mass_matrix has a distribution, but the number of rows is not "
8631 << "equal to the number of degrees of freedom in the problem.";
8632 throw OomphLibError(
8634 }
8635
8636 if (main_matrix.nrow() != this->ndof())
8637 {
8638 std::ostringstream error_stream;
8640 << "main_matrix has a distribution, but the number of rows is not "
8641 << "equal to the number of degrees of freedom in the problem.";
8642 throw OomphLibError(
8644 }
8645 }
8646 // If the distributions are not the same, then complain
8647 else if (main_matrix.distribution_built() !=
8648 mass_matrix.distribution_built())
8649 {
8650 std::ostringstream error_stream;
8651 error_stream << "The distribution of the jacobian and mass matrix must "
8652 << "both be setup or both not setup";
8653 throw OomphLibError(
8655 }
8656#endif
8657
8658 // Store the old assembly handler
8660 // Now setup the eigenproblem handler, pass in the value of the shift
8662
8663 // Prepare the storage formats.
8666 Vector<double*> value(2);
8667 Vector<unsigned> nnz(2);
8668 // Allocate pointer to residuals, although not used in these problems
8670
8671 // determine the distribution for the jacobian (main matrix)
8672 // IF the jacobian has distribution setup then use that
8673 // ELSE determine the distribution based on the
8674 // distributed_matrix_distribution enum
8676 if (main_matrix.distribution_built())
8677 {
8678 dist_pt = new LinearAlgebraDistribution(main_matrix.distribution_pt());
8679 }
8680 else
8681 {
8683 }
8684
8685
8686 // The matrix is in compressed row format
8687 bool compressed_row_flag = true;
8688
8689#ifdef OOMPH_HAS_MPI
8690 //
8691 if (Communicator_pt->nproc() == 1)
8692 {
8693#endif
8694
8697 value,
8698 nnz,
8701
8702 // The main matrix is the first entry
8703 main_matrix.build(dist_pt);
8704 main_matrix.build_without_copy(dist_pt->nrow(),
8705 nnz[0],
8706 value[0],
8709 // The mass matrix is the second entry
8710 mass_matrix.build(dist_pt);
8711 mass_matrix.build_without_copy(dist_pt->nrow(),
8712 nnz[1],
8713 value[1],
8716#ifdef OOMPH_HAS_MPI
8717 }
8718 else
8719 {
8720 if (dist_pt->distributed())
8721 {
8725 value,
8726 nnz,
8728 // The main matrix is the first entry
8729 main_matrix.build(dist_pt);
8730 main_matrix.build_without_copy(dist_pt->nrow(),
8731 nnz[0],
8732 value[0],
8735 // The mass matrix is the second entry
8736 mass_matrix.build(dist_pt);
8737 mass_matrix.build_without_copy(dist_pt->nrow(),
8738 nnz[1],
8739 value[1],
8742 }
8743 else
8744 {
8750 value,
8751 nnz,
8753 // The main matrix is the first entry
8755 main_matrix.build_without_copy(dist_pt->nrow(),
8756 nnz[0],
8757 value[0],
8760 main_matrix.redistribute(dist_pt);
8761 // The mass matrix is the second entry
8763 mass_matrix.build_without_copy(dist_pt->nrow(),
8764 nnz[1],
8765 value[1],
8768 mass_matrix.redistribute(dist_pt);
8769 delete temp_dist_pt;
8770 }
8771 }
8772#endif
8773
8774 // clean up dist_pt and residuals_vector pt
8775 delete dist_pt;
8776
8777 // Delete the eigenproblem handler
8778 delete Assembly_handler_pt;
8779 // Reset the assembly handler to the original handler
8781 }
8782
8783
8784 //=======================================================================
8785 /// Stored the current values of the dofs
8786 //=======================================================================
8788 {
8789 // If memory has not been allocated, then allocated memory for the saved
8790 // dofs
8791 if (Saved_dof_pt == 0)
8792 {
8794 }
8795
8796#ifdef OOMPH_HAS_MPI
8797 // If the problem is distributed I have to do something different
8799 {
8800 // How many entries do we store locally?
8801 const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8802
8803 // Resize the vector
8804 Saved_dof_pt->resize(n_row_local);
8805
8806 // Back 'em up
8807 for (unsigned i = 0; i < n_row_local; i++)
8808 {
8809 (*Saved_dof_pt)[i] = *(this->Dof_pt[i]);
8810 }
8811 }
8812 // Otherwise just store all the dofs
8813 else
8814#endif
8815 {
8816 // Find the number of dofs
8817 unsigned long n_dof = ndof();
8818
8819 // Resize the vector
8820 Saved_dof_pt->resize(n_dof);
8821
8822 // Transfer the values over
8823 for (unsigned long n = 0; n < n_dof; n++)
8824 {
8825 (*Saved_dof_pt)[n] = dof(n);
8826 }
8827 }
8828 }
8829
8830 //====================================================================
8831 /// Restore the saved dofs
8832 //====================================================================
8834 {
8835 // Check that we can do this
8836 if (Saved_dof_pt == 0)
8837 {
8838 throw OomphLibError(
8839 "There are no stored values, use store_current_dof_values()\n",
8842 }
8843
8844
8845#ifdef OOMPH_HAS_MPI
8846 // If the problem is distributed I have to do something different
8848 {
8849 // How many entries do we store locally?
8850 const unsigned n_row_local = Dof_distribution_pt->nrow_local();
8851
8852 if (Saved_dof_pt->size() != n_row_local)
8853 {
8854 throw OomphLibError("The number of stored values is not equal to the "
8855 "current number of dofs\n",
8858 }
8859
8860 // Transfer the values over
8861 for (unsigned long n = 0; n < n_row_local; n++)
8862 {
8863 *(this->Dof_pt[n]) = (*Saved_dof_pt)[n];
8864 }
8865 }
8866 // Otherwise just restore all the dofs
8867 else
8868#endif
8869 {
8870 // Find the number of dofs
8871 unsigned long n_dof = ndof();
8872
8873 if (Saved_dof_pt->size() != n_dof)
8874 {
8875 throw OomphLibError("The number of stored values is not equal to the "
8876 "current number of dofs\n",
8879 }
8880
8881 // Transfer the values over
8882 for (unsigned long n = 0; n < n_dof; n++)
8883 {
8884 dof(n) = (*Saved_dof_pt)[n];
8885 }
8886 }
8887
8888 // Delete the memory
8889 delete Saved_dof_pt;
8890 Saved_dof_pt = 0;
8891 }
8892
8893 //======================================================================
8894 /// Assign the eigenvector passed to the function to the dofs
8895 //======================================================================
8897 {
8898 unsigned long n_dof = ndof();
8899 // Check that the eigenvector has the correct size
8900 if (eigenvector.nrow() != n_dof)
8901 {
8902 std::ostringstream error_message;
8903 error_message << "Eigenvector has size " << eigenvector.nrow()
8904 << ", not equal to the number of dofs in the problem,"
8905 << n_dof << std::endl;
8906
8907 throw OomphLibError(
8908 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8909 }
8910
8911 // Ensure that the eigenvector distribution matches the dof distribution
8912 // Copy vector
8914 // Redistribute the copy to the dof distribution
8915 eigenvector_dof.redistribute(this->Dof_distribution_pt);
8916
8917 // Loop over the dofs and assign the eigenvector
8918 for (unsigned long n = 0; n < eigenvector_dof.nrow_local(); n++)
8919 {
8920 dof(n) = eigenvector_dof[n];
8921 }
8922// Of course we now need to synchronise
8923#ifdef OOMPH_HAS_MPI
8924 this->synchronise_all_dofs();
8925#endif
8926 }
8927
8928
8929 //======================================================================
8930 /// Add the eigenvector passed to the function to the dofs with
8931 /// magnitude epsilon
8932 //======================================================================
8933 void Problem::add_eigenvector_to_dofs(const double& epsilon,
8935 {
8936 unsigned long n_dof = ndof();
8937 // Check that the eigenvector has the correct size
8938 if (eigenvector.nrow() != n_dof)
8939 {
8940 std::ostringstream error_message;
8941 error_message << "Eigenvector has size " << eigenvector.nrow()
8942 << ", not equal to the number of dofs in the problem,"
8943 << n_dof << std::endl;
8944
8945 throw OomphLibError(
8946 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
8947 }
8948
8949 // Ensure that the eigenvector distribution matches the dof distribution
8950 // Copy vector
8952 // Redistribute the copy to the dof distribution
8953 eigenvector_dof.redistribute(this->Dof_distribution_pt);
8954
8955
8956 // Loop over the dofs and add the eigenvector
8957 // Only use local values
8958 for (unsigned long n = 0; n < eigenvector.nrow_local(); n++)
8959 {
8960 dof(n) += epsilon * eigenvector[n];
8961 }
8962// Of course we now need to synchronise
8963#ifdef OOMPH_HAS_MPI
8964 this->synchronise_all_dofs();
8965#endif
8966 }
8967
8968
8969 //================================================================
8970 /// General Newton solver. Requires only a convergence tolerance.
8971 /// The linear solver takes a pointer to the problem (which defines
8972 /// the Jacobian \b J and the residual Vector \b r) and returns
8973 /// the solution \b x of the system
8974 /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
8975 //================================================================
8977 {
8978 // Initialise timers
8979 double total_linear_solver_time = 0.0;
8980 double t_start = TimingHelpers::timer();
8981 Max_res.clear();
8982
8983 // Find total number of dofs
8984 unsigned long n_dofs = ndof();
8985
8986 // Set up the Vector to hold the solution
8988
8989 //-----Variables for the globally convergent Newton method------
8990
8991 // Set up the vector to hold the gradient
8993
8994 // Other variables
8995 double half_residual_squared = 0.0;
8996 double max_step = 0.0;
8997
8998 //--------------------------------------------------------------
8999
9000 // Set the counter
9001 unsigned count = 0;
9002 // Set the loop flag
9003 unsigned LOOP_FLAG = 1;
9004
9006 {
9007#ifdef OOMPH_HAS_MPI
9008 // Break if running in parallel
9010 {
9011 std::ostringstream error_stream;
9012 error_stream << "Globally convergent Newton method has not been "
9013 << "implemented in parallel yet!" << std::endl;
9014 throw OomphLibError(
9016 }
9017#endif
9018
9019 // Get gradient
9021 // Reset the gradient (clear it), since the number of dofs and
9022 // hence the size of the DoubleVector might have changed
9024 }
9025
9026 // Update anything that needs updating
9028
9029 // Reset number of Newton iterations taken
9031
9032 // Now do the Newton loop
9033 do
9034 {
9035 count++;
9036
9037 // Do any updates that are required
9039
9040
9041 // No degrees of freedom? What are you solving for?
9042 if (n_dofs == 0)
9043 {
9044 oomph_info << std::endl << std::endl << std::endl;
9045 oomph_info << "This is a bit bizarre: The problem has no dofs."
9046 << std::endl;
9048 << "I'll just return from the Newton solver without doing anything."
9049 << std::endl;
9050
9051 // Do any updates that would have been performed
9056
9057 oomph_info << "I hope this is what you intended me to do..."
9058 << std::endl;
9059 oomph_info << std::endl
9060 << "Note: All actions_...() functions were called"
9061 << std::endl;
9062 oomph_info << std::endl << " before returning." << std::endl;
9063 oomph_info << std::endl << std::endl << std::endl;
9064 return;
9065 }
9066
9067 // Calculate initial residuals
9068 if (count == 1)
9069 {
9070 // Is the problem nonlinear? If not ignore the pre-iteration
9071 // convergence check.
9073 {
9074#ifdef OOMPH_HAS_MPI
9075 // Synchronise the solution on different processors (on each submesh)
9076 this->synchronise_all_dofs();
9077#endif
9078
9080 dx.clear();
9082
9083 // Get half of squared residual and find maximum step length
9084 // for step length control
9086 {
9088 double sum = 0.0;
9089 for (unsigned i = 0; i < n_dofs; i++)
9090 {
9091 sum += (*Dof_pt[i]) * (*Dof_pt[i]);
9092 half_residual_squared += dx[i] * dx[i];
9093 }
9094 half_residual_squared *= 0.5;
9095 max_step = 100.0 * std::max(sqrt(sum), double(n_dofs));
9096 }
9097
9098 // Get maximum residuals
9099 double maxres = dx.max();
9100 Max_res.push_back(maxres);
9101
9103 {
9104 // Let's output the residuals
9105 // unsigned n_row_local = dx.distribution_pt()->nrow_local();
9106 // unsigned first_row = dx.distribution_pt()->first_row();
9107 // for(unsigned n=0;n<n_row_local;n++)
9108 //{
9109 // oomph_info << "residual: " << n + first_row << " " << dx[n] <<
9110 // "\n";
9111 //}
9112
9113 oomph_info << "\nInitial Maximum residuals " << maxres << std::endl;
9114 }
9115
9116 if ((maxres < Newton_solver_tolerance) &&
9118 {
9119 LOOP_FLAG = 0;
9120 continue;
9121 }
9122 }
9123 else
9124 {
9126 {
9128 << "Linear problem -- convergence in one iteration assumed."
9129 << std::endl;
9130 }
9131 }
9132 }
9133
9134
9135 // Increment number of Newton iterations taken
9137
9138 // Initialise timer for linear solver
9140
9141 // Now do the linear solve -- recycling Jacobian if requested
9143 {
9145 {
9146 oomph_info << "Not recomputing Jacobian! " << std::endl;
9147 }
9148
9149 // If we're doing the first iteration and the problem is nonlinear,
9150 // the residuals have already been computed above during the
9151 // initial convergence check. Otherwise compute them here.
9152 if ((count != 1) || (!Problem_is_nonlinear)) get_residuals(dx);
9153
9154 // Backup residuals
9156
9157 // Resolve
9159 }
9160 else
9161 {
9163 {
9165 {
9166 oomph_info << "Enabling resolve" << std::endl;
9167 }
9169 }
9170 Linear_solver_pt->solve(this, dx);
9172 }
9173
9174 // End of linear solver
9177
9179 {
9180 oomph_info << std::endl;
9181 oomph_info << "Time for linear solver (ndof=" << n_dofs << "): "
9184 << std::endl
9185 << std::endl;
9186 }
9187
9188 // Subtract the new values from the true dofs
9189 dx.redistribute(Dof_distribution_pt);
9190 double* dx_pt = dx.values_pt();
9192
9194 {
9195 // Get the gradient
9197
9198 for (unsigned i = 0; i < ndof_local; i++)
9199 {
9200 dx_pt[i] *= -1.0;
9201 }
9202
9203 // Update with steplength control
9205
9206 for (unsigned i = 0; i < ndof_local; i++)
9207 {
9208 unknowns_old[i] = *Dof_pt[i];
9209 }
9210
9214 gradient,
9215 dx,
9217 max_step);
9218 }
9219 // direct Newton update
9220 else
9221 {
9222 for (unsigned l = 0; l < ndof_local; l++)
9223 {
9225 }
9226 }
9227#ifdef OOMPH_HAS_MPI
9228 // Synchronise the solution on different processors (on each submesh)
9229 this->synchronise_all_dofs();
9230#endif
9231
9232 // Do any updates that are required
9235
9236 // Maximum residuals
9237 double maxres = 0.0;
9238 // If the user has declared that the Problem is linear
9239 // we ignore the convergence check
9241 {
9242 // Get the maximum residuals
9243 // maxres = std::fabs(*std::max_element(dx.begin(),dx.end(),
9244 // AbsCmp<double>()));
9245 // oomph_info << "Maxres correction " << maxres << "\n";
9246
9247 // Calculate the new residuals
9248 dx.clear();
9250
9251 // Get the maximum residuals
9252 maxres = dx.max();
9253 Max_res.push_back(maxres);
9254
9256 {
9257 oomph_info << "Newton Step " << count << ": Maximum residuals "
9258 << maxres << std::endl
9259 << std::endl;
9260 }
9261 }
9262
9263 // If we have converged jump straight to the test at the end of the loop
9264 if (maxres < Newton_solver_tolerance)
9265 {
9266 LOOP_FLAG = 0;
9267 continue;
9268 }
9269
9270 // This section will not be reached if we have converged already
9271 // If the maximum number of residuals is too high or the maximum number
9272 // of iterations has been reached
9273 if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9274 {
9275 // Print a warning -- regardless of what the throw does
9276 if (maxres > Max_residuals)
9277 {
9278 oomph_info << "Max. residual (" << Max_residuals
9279 << ") has been exceeded in Newton solver." << std::endl;
9280 }
9282 {
9283 oomph_info << "Reached max. number of iterations ("
9284 << Max_newton_iterations << ") in Newton solver."
9285 << std::endl;
9286 }
9287 // Now throw...
9288 throw NewtonSolverError(count, maxres);
9289 }
9290
9291 } while (LOOP_FLAG);
9292
9293 // Now update anything that needs updating
9295
9296 // Finalise/doc timings
9298 {
9299 oomph_info << std::endl;
9300 oomph_info << "Total time for linear solver (ndof=" << n_dofs << "): "
9303 << std::endl;
9304 }
9305
9306 double t_end = TimingHelpers::timer();
9307 double total_time = t_end - t_start;
9308
9310 {
9311 oomph_info << "Total time for Newton solver (ndof=" << n_dofs << "): "
9313 << std::endl;
9314 }
9315 if (total_time > 0.0)
9316 {
9318 {
9319 oomph_info << "Time outside linear solver : "
9321 100.0
9322 << " %" << std::endl;
9323 }
9324 }
9325 else
9326 {
9328 {
9329 oomph_info << "Time outside linear solver : "
9330 << "[too fast]" << std::endl;
9331 }
9332 }
9333 if (!Shut_up_in_newton_solve) oomph_info << std::endl;
9334 }
9335
9336 //========================================================================
9337 /// Helper function for the globally convergent Newton solver
9338 //========================================================================
9340 const Vector<double>& x_old,
9341 const double& half_residual_squared_old,
9344 double& half_residual_squared,
9345 const double& stpmax)
9346 {
9347 const double min_fct_decrease = 1.0e-4;
9348 double convergence_tol_on_x = 1.0e-16;
9349 double f_aux = 0.0;
9350 double lambda_aux = 0.0;
9351 double proposed_lambda;
9352 unsigned long n_dof = ndof();
9353 double sum = 0.0;
9354 for (unsigned i = 0; i < n_dof; i++)
9355 {
9356 sum += newton_dir[i] * newton_dir[i];
9357 }
9358 sum = sqrt(sum);
9359 if (sum > stpmax)
9360 {
9361 for (unsigned i = 0; i < n_dof; i++)
9362 {
9363 newton_dir[i] *= stpmax / sum;
9364 }
9365 }
9366 double slope = 0.0;
9367 for (unsigned i = 0; i < n_dof; i++)
9368 {
9369 slope += gradient[i] * newton_dir[i];
9370 }
9371 if (slope >= 0.0)
9372 {
9373 std::ostringstream warn_message;
9374 warn_message << "WARNING: Non-negative slope, probably due to a "
9375 << " roundoff \nproblem in the linesearch: slope=" << slope
9376 << "\n";
9378 "Problem::globally_convergent_line_search()",
9380 }
9381 double test = 0.0;
9382 for (unsigned i = 0; i < n_dof; i++)
9383 {
9384 double temp =
9385 std::fabs(newton_dir[i]) / std::max(std::fabs(x_old[i]), 1.0);
9386 if (temp > test) test = temp;
9387 }
9389 double lambda = 1.0;
9390 while (true)
9391 {
9392 for (unsigned i = 0; i < n_dof; i++)
9393 {
9394 *Dof_pt[i] = x_old[i] + lambda * newton_dir[i];
9395 }
9396
9397 // Evaluate current residuals
9401 for (unsigned i = 0; i < n_dof; i++)
9402 {
9404 }
9405 half_residual_squared *= 0.5;
9406
9407 if (lambda < lambda_min)
9408 {
9409 for (unsigned i = 0; i < n_dof; i++) *Dof_pt[i] = x_old[i];
9410
9411 std::ostringstream warn_message;
9412 warn_message << "WARNING: Line search converged on x only!\n";
9414 "Problem::globally_convergent_line_search()",
9416 return;
9417 }
9418 else if (half_residual_squared <=
9420 {
9421 oomph_info << "Returning from linesearch with lambda=" << lambda
9422 << std::endl;
9423 return;
9424 }
9425 else
9426 {
9427 if (lambda == 1.0)
9428 {
9430 -slope /
9432 }
9433 else
9434 {
9435 double r1 =
9438 double a_poly =
9439 (r1 / (lambda * lambda) - r2 / (lambda_aux * lambda_aux)) /
9440 (lambda - lambda_aux);
9441 double b_poly = (-lambda_aux * r1 / (lambda * lambda) +
9442 lambda * r2 / (lambda_aux * lambda_aux)) /
9443 (lambda - lambda_aux);
9444 if (a_poly == 0.0)
9445 {
9446 proposed_lambda = -slope / (2.0 * b_poly);
9447 }
9448 else
9449 {
9450 double discriminant = b_poly * b_poly - 3.0 * a_poly * slope;
9451 if (discriminant < 0.0)
9452 {
9453 proposed_lambda = 0.5 * lambda;
9454 }
9455 else if (b_poly <= 0.0)
9456 {
9457 proposed_lambda = (-b_poly + sqrt(discriminant)) / (3.0 * a_poly);
9458 }
9459 else
9460 {
9462 }
9463 }
9464 if (proposed_lambda > 0.5 * lambda)
9465 {
9466 proposed_lambda = 0.5 * lambda;
9467 }
9468 }
9469 }
9470 lambda_aux = lambda;
9472 lambda = std::max(proposed_lambda, 0.1 * lambda);
9473 }
9474 }
9475
9476
9477 //========================================================================
9478 /// Solve a steady problem, in the context of an overall unsteady problem.
9479 /// This is achieved by setting the weights in the timesteppers to be zero
9480 /// which has the effect of rendering them steady timesteppers
9481 /// The optional argument max_adapt specifies the max. number of
9482 /// adaptations of all refineable submeshes are performed to
9483 /// achieve the the error targets specified in the refineable submeshes.
9484 //========================================================================
9486 {
9487 // Find out how many timesteppers there are
9488 unsigned n_time_steppers = ntime_stepper();
9489
9490 // Vector of bools to store the is_steady status of the various
9491 // timesteppers when we came in here
9492 std::vector<bool> was_steady(n_time_steppers);
9493
9494 // Loop over them all and make them (temporarily) static
9495 for (unsigned i = 0; i < n_time_steppers; i++)
9496 {
9499 }
9500
9501 try
9502 {
9503 // Solve the non-linear problem with Newton's method
9504 if (max_adapt == 0)
9505 {
9506 newton_solve();
9507 }
9508 else
9509 {
9511 }
9512 }
9513 // Catch any exceptions thrown in the Newton solver
9514 catch (NewtonSolverError& error)
9515 {
9516 oomph_info << std::endl
9517 << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
9518 // Check whether it's the linear solver
9519 if (error.linear_solver_error)
9520 {
9521 oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
9522 }
9523 // Check to see whether we have reached Max_iterations
9524 else if (error.iterations == Max_newton_iterations)
9525 {
9526 oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
9527 << ") REACHED WITHOUT CONVERGENCE " << std::endl;
9528 }
9529 // If not, it must be that we have exceeded the maximum residuals
9530 else
9531 {
9532 oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
9533 << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
9534 << std::endl;
9535 }
9536
9537 // Die horribly!!
9538 std::ostringstream error_stream;
9539 error_stream << "Error occured in Newton solver. " << std::endl;
9540 throw OomphLibError(
9542 }
9543
9544
9545 // Reset the is_steady status of all timesteppers that
9546 // weren't already steady when we came in here and reset their
9547 // weights
9548 for (unsigned i = 0; i < n_time_steppers; i++)
9549 {
9550 if (!was_steady[i])
9551 {
9553 }
9554 }
9555
9556 // Since we performed a steady solve, the history values
9557 // now have to be set as if we had performed an impulsive start from
9558 // the current solution. This ensures that the time-derivatives
9559 // evaluate to zero even now that the timesteppers have been
9560 // reactivated.
9562 }
9563
9564 //===========================================================================
9565 /// Perform a basic continuation step using Newton's method. The governing
9566 /// parameter of the problem is passed as a pointer to the routine. The
9567 /// number of Newton steps taken is returned
9568 //==========================================================================
9570 {
9571 // Set up memory for z
9572 // unsigned long n_dofs = ndof();
9573 // LinearAlgebraDistribution dist(Communicator_pt,n_dofs,false);
9574 // DoubleVector z(&dist,0.0);
9575 DoubleVector z;
9576 // Call the solver
9578 }
9579
9580
9581 //===================================================================
9582 /// This function performs a basic continuation step using the Newton method.
9583 /// The number of Newton steps taken is returned, to be used in any
9584 /// external step-size control routines.
9585 /// The governing parameter of the problem is passed as a pointer to the
9586 /// routine, as is the sign of the Jacobian and a Vector in which
9587 /// to store the derivatives wrt the parameter, if required.
9588 //==================================================================
9590 DoubleVector& z)
9591 {
9592 // Find the total number of dofs
9593 // unsigned long n_dofs = ndof();
9594
9595 // Find the local number of dofs
9597
9598 // create the distribution (not distributed)
9599 // LinearAlgebraDistribution dist(this->communicator_pt(),n_dofs,false);
9600
9601 // Assign memory for solutions of the equations
9602 // DoubleVector y(&dist,0.0);
9603 DoubleVector y;
9604
9605 // Assign memory for the dot products of the uderivatives and y and z
9606 double uderiv_dot_y = 0.0, uderiv_dot_z = 0.0;
9607 // Set and initialise the counter
9608 unsigned count = 0;
9609 // Set the loop flag
9610 unsigned LOOP_FLAG = 1;
9611
9612 // Update anything that needs updating
9614
9615 // Check the arc-length constraint
9616 double arc_length_constraint_residual = 0.0;
9617
9618 // Are we storing the matrix in the linear solve
9619 bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9620
9621 // For this problem, we must store the residuals
9623
9624 // Now do the Newton loop
9625 do
9626 {
9627 count++;
9628
9629 // Do any updates that are required
9631
9632 // Calculate initial residuals
9633 if (count == 1)
9634 {
9635#ifdef OOMPH_HAS_MPI
9636 // Synchronise the solution on different processors (on each submesh)
9637 this->synchronise_all_dofs();
9638#endif
9639
9641 y.clear();
9642 get_residuals(y);
9643 // Get maximum residuals, using our own abscmp function
9644 double maxres = y.max();
9645
9646 // Assemble the residuals for the arc-length step
9648 // Add the variables
9649 for (unsigned long l = 0; l < ndof_local; l++)
9650 {
9653 }
9654
9655 // Now reduce if we have been distributed
9656#ifdef OOMPH_HAS_MPI
9659 (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9660 {
9663 1,
9664 MPI_DOUBLE,
9665 MPI_SUM,
9666 Dof_distribution_pt->communicator_pt()->mpi_comm());
9667 }
9669#endif
9670
9674 Ds_current;
9675
9676 // Is it the max
9677 if (std::fabs(arc_length_constraint_residual) > maxres)
9678 {
9679 maxres = std::fabs(arc_length_constraint_residual);
9680 }
9681
9682 // Find the max
9684 {
9685 oomph_info << "Initial Maximum residuals " << maxres << std::endl;
9686 }
9687
9688 // If we are below the Tolerance, then return immediately
9689 if ((maxres < Newton_solver_tolerance) &&
9691 {
9692 LOOP_FLAG = 0;
9693 count = 0;
9694 continue;
9695 }
9696 }
9697
9698 // If it's the block hopf solver we need to solve for both rhs's
9699 // simultaneously. This is because the block decomposition involves
9700 // solves with two different matrices and storing both at once to
9701 // allow general resolves would be more expensive than necessary.
9702 if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9703 {
9704 // Get the vector dresiduals/dparameter
9705 z.clear();
9707
9708 // Copy rhs vector into local storage so it doesn't get overwritten
9709 // if the linear solver decides to initialise the solution vector, say,
9710 // which it's quite entitled to do!
9712
9713 // Solve the system for the two right-hand sides.
9715 ->solve_for_two_rhs(this, y, input_z, z);
9716 }
9717 // Otherwise
9718 else
9719 {
9720 // Solve the standard problem
9721 Linear_solver_pt->solve(this, y);
9722
9723 // Get the vector dresiduals/dparameter
9724 z.clear();
9726
9727 // Copy rhs vector into local storage so it doesn't get overwritten
9728 // if the linear solver decides to initialise the solution vector, say,
9729 // which it's quite entitled to do!
9731
9732 // Redistribute the RHS to match the linear solver
9733 // input_z.redistribute(Linear_solver_pt->distribution_pt());
9734 // Do not clear z because we assume that it has dR/dparam
9735 z.clear();
9736 // Now resolve the system with the new RHS
9738 }
9739
9740 // Redistribute the results into the natural distribution
9743
9744 // Now we need to calculate dparam, for which we must calculate the
9745 // dot product of the derivatives and y and z
9746 // Reset these values to zero
9747 uderiv_dot_y = 0.0;
9748 uderiv_dot_z = 0.0;
9749 // Now calculate the dot products of the derivative and the solutions
9750 // of the linear system
9751 // Cache pointers to the data in the distributed vectors
9752 double* const y_pt = y.values_pt();
9753 double* const z_pt = z.values_pt();
9754 for (unsigned long l = 0; l < ndof_local; l++)
9755 {
9758 }
9759
9760 // Now reduce if we have been distributed
9761#ifdef OOMPH_HAS_MPI
9762 // Create send and receive arrays of size two
9763 double uderiv_dot[2];
9764 double uderiv_dot2[2];
9769 // Now reduce both together
9771 (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9772 {
9775 2,
9776 MPI_DOUBLE,
9777 MPI_SUM,
9778 Dof_distribution_pt->communicator_pt()->mpi_comm());
9779 }
9782#endif
9783
9784 // Now scale the results
9787
9788 // The set the change in the parameter, given by the pseudo-arclength
9789 // equation. Note that here we are assuming that the arc-length
9790 // equation is always exactly zero,
9791 // which seems to work OK, and saves on some storage.
9792 // In fact, it's more subtle than that. If we include this
9793 // proper residual then we will have to solve the eigenproblem.
9794 // This will make the solver more robust and *should* be done
9795 // ... at some point.
9798
9799 // Set the new value of the parameter
9800 *parameter_pt -= dparam;
9801
9802 // Update the values of the other degrees of freedom
9803 for (unsigned long l = 0; l < ndof_local; l++)
9804 {
9805 *Dof_pt[l] -= y_pt[l] - dparam * z_pt[l];
9806 }
9807
9808 // Calculate the new residuals
9809#ifdef OOMPH_HAS_MPI
9810 // Synchronise the solution on different processors (on each submesh)
9811 this->synchronise_all_dofs();
9812#endif
9813
9814 // Do any updates that are required
9817
9818 y.clear();
9819 get_residuals(y);
9820
9821 // Get the maximum residuals
9822 double maxres = y.max();
9823
9824 // Assemble the residuals for the arc-length step
9826 // Add the variables
9827 for (unsigned long l = 0; l < ndof_local; l++)
9828 {
9831 }
9832
9833 // Now reduce if we have been distributed
9834#ifdef OOMPH_HAS_MPI
9837 (Dof_distribution_pt->communicator_pt()->nproc() > 1))
9838 {
9841 1,
9842 MPI_DOUBLE,
9843 MPI_SUM,
9844 Dof_distribution_pt->communicator_pt()->mpi_comm());
9845 }
9847#endif
9848
9852
9853 // Is it the max
9854 if (std::fabs(arc_length_constraint_residual) > maxres)
9855 {
9856 maxres = std::fabs(arc_length_constraint_residual);
9857 }
9858
9860 {
9861 oomph_info << "Continuation Step " << count << ": Maximum residuals "
9862 << maxres << "\n";
9863 }
9864
9865 // If we have converged jump straight to the test at the end of the loop
9866 if (maxres < Newton_solver_tolerance)
9867 {
9868 LOOP_FLAG = 0;
9869 continue;
9870 }
9871
9872 // This section will not be reached if we have converged already
9873 // If the maximum number of residuals is too high or the maximum number
9874 // of iterations has been reached
9875 if ((maxres > Max_residuals) || (count == Max_newton_iterations))
9876 {
9877 throw NewtonSolverError(count, maxres);
9878 }
9879
9880 } while (LOOP_FLAG);
9881
9882 // Now update anything that needs updating
9884
9885 // Reset the storage of the matrix on the linear solver to what it was
9886 // on entry to this routine
9887 if (enable_resolve)
9888 {
9890 }
9891 else
9892 {
9894 }
9895
9896 // Return the number of Newton Steps taken
9897 return count;
9898 }
9899
9900 //=========================================================================
9901 /// A function to calculate the derivatives wrt the arc-length. This version
9902 /// of the function actually does a linear solve so that the derivatives
9903 /// are calculated "exactly" rather than using the values at the Newton
9904 /// step just before convergence. This is only necessary in spatially adaptive
9905 /// problems, in which the number of degrees of freedom changes and so
9906 /// the appropriate derivatives must be calculated for the new variables.
9907 //=========================================================================
9909 {
9910 // Find the number of degrees of freedom in the problem
9911 const unsigned long n_dofs = ndof();
9912
9913 // create a non-distributed z vector
9915
9916 // Assign memory for solutions of the equations
9917 DoubleVector z(&dist, 0.0);
9918
9919 // If it's the block hopf solver need to solve for both RHS
9920 // at once, but this would all be alleviated if we have the solve
9921 // for the non-residuals RHS.
9922 if (dynamic_cast<BlockHopfLinearSolver*>(Linear_solver_pt))
9923 {
9924 // Get the vector dresiduals/dparameter
9926
9927 // Copy rhs vector into local storage so it doesn't get overwritten
9928 // if the linear solver decides to initialise the solution vector, say,
9929 // which it's quite entitled to do!
9930 DoubleVector dummy(&dist, 0.0), input_z(z);
9931
9932 // Solve for the two RHSs
9934 ->solve_for_two_rhs(this, dummy, input_z, z);
9935 }
9936 // Otherwise we can use the normal resolve
9937 else
9938 {
9939 // Save the status before entry to this routine
9940 bool enable_resolve = Linear_solver_pt->is_resolve_enabled();
9941
9942 // We need to do resolves
9944
9945 // Solve the standard problem, we only want to make sure that
9946 // we factorise the matrix, if it has not been factorised. We shall
9947 // ignore the return value of z.
9948 Linear_solver_pt->solve(this, z);
9949
9950 // Get the vector dresiduals/dparameter
9952
9953
9954 // Copy rhs vector into local storage so it doesn't get overwritten
9955 // if the linear solver decides to initialise the solution vector, say,
9956 // which it's quite entitled to do!
9958
9959 // Now resolve the system with the new RHS and overwrite the solution
9961
9962 // Restore the storage status of the linear solver
9963 if (enable_resolve)
9964 {
9966 }
9967 else
9968 {
9970 }
9971 }
9972
9973 // Now, we can calculate the derivatives, etc
9975 }
9976
9977 //=======================================================================
9978 /// A function to calculate the derivatives with respect to the arc-length
9979 /// required for continuation. The arguments is the solution of the
9980 /// linear system,
9981 /// Jz = dR/dparameter, that gives du/dparameter and the direction
9982 /// output from the newton_solve_continuation function. The derivatives
9983 /// are stored in the ContinuationParameters namespace.
9984 //===================================================================
9986 {
9987 // Calculate the continuation derivatives
9989
9990 // Scale the value of theta if the control flag is set
9991 if (Scale_arc_length)
9992 {
9993 // Don't divide by zero!
9994 if (Parameter_derivative != 1.0)
9995 {
10000
10001 // Recalculate the continuation derivatives with the new scaled values
10003 }
10004 }
10005 }
10006
10007 //=======================================================================
10008 /// A function to calculate the derivatives with respect to the arc-length
10009 /// required for continuation using finite differences.
10010 //===================================================================
10012 double* const& parameter_pt)
10013 {
10014 // Calculate the continuation derivatives
10016
10017 // Scale the value of theta if the control flag is set
10018 if (Scale_arc_length)
10019 {
10020 // Don't divide by zero!
10021 if (Parameter_derivative != 1.0)
10022 {
10027
10028 // Recalculate the continuation derivatives with the new scaled values
10030 }
10031 }
10032 }
10033
10034 //======================================================================
10035 /// Function that returns a boolean flag to indicate whether the pointer
10036 /// parameter_pt refers to memory that is a value in a Data object used
10037 /// within the problem
10038 //======================================================================
10040 double* const& parameter_pt)
10041 {
10042 // Firstly check the global data
10043 const unsigned n_global = Global_data_pt.size();
10044 for (unsigned i = 0; i < n_global; ++i)
10045 {
10046 // If we find it then return true
10047 if (Global_data_pt[i]->does_pointer_correspond_to_value(parameter_pt))
10048 {
10049 return true;
10050 }
10051 }
10052
10053 // If we find the pointer in the mesh data return true
10055 {
10056 return true;
10057 }
10058
10059 // Loop over the submeshes to handle the case of spine data
10060 const unsigned n_sub_mesh = this->nsub_mesh();
10061 // If there is only one mesh
10062 if (n_sub_mesh == 0)
10063 {
10064 if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
10065 {
10066 if (spine_mesh_pt->does_pointer_correspond_to_spine_data(parameter_pt))
10067 {
10068 return true;
10069 }
10070 }
10071 }
10072 // Otherwise loop over the sub meshes
10073 else
10074 {
10075 // Assign global equation numbers first
10076 for (unsigned i = 0; i < n_sub_mesh; i++)
10077 {
10078 if (SpineMesh* const spine_mesh_pt =
10079 dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
10080 {
10081 if (spine_mesh_pt->does_pointer_correspond_to_spine_data(
10082 parameter_pt))
10083 {
10084 return true;
10085 }
10086 }
10087 }
10088 }
10089
10090 // If we have got here then the data is not stored in the problem, so return
10091 // false
10092 return false;
10093 }
10094
10095
10096 //=======================================================================
10097 /// A private helper function to
10098 /// calculate the derivatives with respect to the arc-length
10099 /// required for continuation. The arguments is the solution of the
10100 /// linear system,
10101 /// Jz = dR/dparameter, that gives du/dparameter and the direction
10102 /// output from the newton_solve_continuation function. The derivatives
10103 /// are stored in the ContinuationParameters namespace.
10104 //===================================================================
10106 {
10107 // Find the number of degrees of freedom in the problem
10108 // unsigned long n_dofs = ndof();
10109 // Find the number of local dofs in the problem
10110 const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
10111
10112 // Work out the continuation direction
10113 // The idea is that (du/ds)_{old} . (du/ds)_{new} >= 0
10114 // if the direction is to remain the same.
10115 // du/ds_{new} = [dlambda/ds; du/ds] = [dlambda/ds ; - dlambda/ds z]
10116 // so (du/ds)_{new} . (du/ds)_{old}
10117 // = dlambda/ds [1 ; - z] . [ Parameter_derivative ; Dof_derivatives]
10118 // = dlambda/ds (Parameter_derivative - Dof_derivative . z)
10119
10120 // Create a local copy of z that can be redistributed without breaking
10121 // the constness of z
10123
10124 // Redistribute z so that it has the (natural) dof distribution
10125 local_z.redistribute(Dof_distribution_pt);
10126
10127 // Calculate the local contribution to the Continuation direction
10129 // Cache the pointer to z
10130 double* const local_z_pt = local_z.values_pt();
10131 for (unsigned long l = 0; l < ndof_local; l++)
10132 {
10134 }
10135
10136 // Now reduce if we have been distributed
10137#ifdef OOMPH_HAS_MPI
10140 (Dof_distribution_pt->communicator_pt()->nproc() > 1))
10141 {
10143 &cont_dir2,
10144 1,
10145 MPI_DOUBLE,
10146 MPI_SUM,
10147 Dof_distribution_pt->communicator_pt()->mpi_comm());
10148 }
10150#endif
10151
10152 // Add parameter derivative
10154
10155 // Calculate the magnitude of the du/ds Vector
10156
10157 // Note that actually, we are usually approximating by using the value at
10158 // newton step just before convergence, which saves one additional
10159 // Newton solve.
10160
10161 // First calculate the magnitude of du/dparameter, chi
10162 double chi = local_z.dot(local_z);
10163
10164 // Calculate the current derivative of the parameter wrt the arc-length
10165 Parameter_derivative = 1.0 / sqrt(1.0 + Theta_squared * chi);
10166
10167 // If the dot product of the current derivative wrt the Direction
10168 // is less than zero, switch the sign of the derivative to ensure
10169 // smooth continuation
10171 {
10172 Parameter_derivative *= -1.0;
10173 }
10174
10175 // Resize the derivatives array, if necessary
10177 {
10178 if (Dof_derivative.size() != ndof_local)
10179 {
10180 Dof_derivative.resize(ndof_local, 0.0);
10181 }
10182 }
10183 // Calculate the new derivatives wrt the arc-length
10184 for (unsigned long l = 0; l < ndof_local; l++)
10185 {
10186 // This comes from the formulation J u_dot + dr/dlambda lambda_dot = 0
10187 // on the curve and then it follows that.
10189 }
10190 }
10191
10192 //=======================================================================
10193 /// A private helper function to
10194 /// calculate the derivatives with respect to the arc-length
10195 /// required for continuation using finite differences.
10196 //===================================================================
10198 double* const& parameter_pt)
10199 {
10200 // Find the number of values
10201 // const unsigned long n_dofs = this->ndof();
10202 // Find the number of local dofs in the problem
10203 const unsigned long ndof_local = Dof_distribution_pt->nrow_local();
10204
10205 // Temporary storage for the finite-difference approximation to the helper
10207 double length = 0.0;
10208 // Calculate the change in values and contribution to total length
10209 for (unsigned long l = 0; l < ndof_local; l++)
10210 {
10211 z[l] = (*Dof_pt[l] - Dof_current[l]) / Ds_current;
10212 length += Theta_squared * z[l] * z[l];
10213 }
10214
10215 // Reduce if parallel
10216#ifdef OOMPH_HAS_MPI
10217 double length2 = length;
10219 (Dof_distribution_pt->communicator_pt()->nproc() > 1))
10220 {
10222 &length2,
10223 1,
10224 MPI_DOUBLE,
10225 MPI_SUM,
10226 Dof_distribution_pt->communicator_pt()->mpi_comm());
10227 }
10228 length = length2;
10229#endif
10230
10231 // Calculate change in parameter
10233 length += Z * Z;
10234
10235 // Scale the approximations to the derivatives
10236 length = sqrt(length);
10237 for (unsigned long l = 0; l < ndof_local; l++)
10238 {
10239 dof_derivative(l) = z[l] / length;
10240 }
10242 }
10243
10244
10245 /// Virtual function that is used to symmetrise the problem so that
10246 /// the current solution exactly satisfies any symmetries within the system.
10247 /// Used when adpativly solving pitchfork detection problems when small
10248 /// asymmetries in the coarse solution can be magnified
10249 /// leading to very inaccurate answers on the fine mesh.
10250 /// This is always problem-specific and must be filled in by the user
10251 /// The default issues a warning
10253 {
10254 std::ostringstream warn_message;
10256 << "Warning: This function is called after spatially adapting the\n"
10257 << "eigenfunction associated with a pitchfork bifurcation and should\n"
10258 << "ensure that the exact (anti-)symmetries of problem are enforced\n"
10259 << "within that eigenfunction. It is problem specific and must be\n"
10260 << "filled in by the user if required.\n"
10261 << "A sign of problems is if the slack paramter gets too large and\n"
10262 << "if the solution at the Pitchfork is not symmetric.\n";
10264 warn_message.str(),
10265 "Problem::symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()",
10267 }
10268
10269 //====================================================================
10270 /// Return pointer to the parameter that is used in the
10271 /// bifurcation detection. If we are not tracking a bifurcation then
10272 /// an error will be thrown by the AssemblyHandler
10273 //====================================================================
10275 {
10277 }
10278
10279 //====================================================================
10280 /// Return the eigenfunction calculated as part of a
10281 /// bifurcation tracking process. If we are not tracking a bifurcation
10282 /// then an error will be thrown by the AssemblyHandler
10283 //======================================================================
10286 {
10287 // Simply call the appropriate assembly handler function
10289 }
10290
10291 //============================================================
10292 /// Activate the fold tracking system by changing the assembly
10293 /// handler and initialising it using the parameter addressed
10294 /// by parameter_pt.
10295 //============================================================
10297 const bool& block_solve)
10298 {
10299 // Reset the assembly handler to default
10301 // Set the new assembly handler. Note that the constructor actually
10302 // solves the original problem to get some initial conditions, but
10303 // this is OK because the RHS is always evaluated before assignment.
10305
10306 // If we are using a block solver, we must set the linear solver pointer
10307 // to the block fold solver. The present linear solver is
10308 // used by the block solver and so must be passed as an argument.
10309 // The destructor of the Fold handler returns the linear
10310 // solver to the original non-block version.
10311 if (block_solve)
10312 {
10314 }
10315 }
10316
10317 //===============================================================
10318 /// Activate the generic bifurcation ///tracking system by changing the
10319 /// assembly handler and initialising it using the parameter addressed by
10320 /// parameter_pt.
10321 //============================================================
10324 const bool& block_solve)
10325 {
10326 // Reset the assembly handler to default
10328 // Set the new assembly handler. Note that the constructor actually
10329 // solves the original problem to get some initial conditions, but
10330 // this is OK because the RHS is always evaluated before assignment.
10332
10333 // If we are using a block solver, we must set the linear solver pointer
10334 // to the block fold solver. The present linear solver is
10335 // used by the block solver and so must be passed as an argument.
10336 // The destructor of the Fold handler returns the linear
10337 // solver to the original non-block version.
10338 if (block_solve)
10339 {
10341 }
10342 }
10343
10344
10345 //===============================================================
10346 /// Activate the generic bifurcation ///tracking system by changing the
10347 /// assembly handler and initialising it using the parameter addressed by
10348 /// parameter_pt.
10349 //============================================================
10353 const bool& block_solve)
10354 {
10355 // Reset the assembly handler to default
10357 // Set the new assembly handler. Note that the constructor actually
10358 // solves the original problem to get some initial conditions, but
10359 // this is OK because the RHS is always evaluated before assignment.
10362
10363 // If we are using a block solver, we must set the linear solver pointer
10364 // to the block fold solver. The present linear solver is
10365 // used by the block solver and so must be passed as an argument.
10366 // The destructor of the Fold handler returns the linear
10367 // solver to the original non-block version.
10368 if (block_solve)
10369 {
10371 }
10372 }
10373
10374
10375 //==================================================================
10376 /// Activate the pitchfork tracking system by changing the assembly
10377 /// handler and initialising it using the parameter addressed
10378 /// by parameter_pt and a symmetry vector. The boolean flag is
10379 /// used to specify whether a block solver is used, default is true.
10380 //===================================================================
10383 const bool& block_solve)
10384 {
10385 // Reset the assembly handler to default
10387
10388 // Set the new assembly handler. Note that the constructor actually
10389 // solves the original problem to get some initial conditions, but
10390 // this is OK because the RHS is always evaluated before assignment.
10392 this, this->assembly_handler_pt(), parameter_pt, symmetry_vector);
10393
10394 // If we are using a block solver, we must set the linear solver pointer
10395 // to the block pitchfork solver. The present linear solver is
10396 // used by the block solver and so must be passed as an argument.
10397 // The destructor of the PitchFork handler returns the linear
10398 // solver to the original non-block version.
10399 if (block_solve)
10400 {
10402 }
10403 }
10404
10405
10406 //============================================================
10407 /// Activate the hopf tracking system by changing the assembly
10408 /// handler and initialising it using the parameter addressed
10409 /// by parameter_pt.
10410 //============================================================
10412 const bool& block_solve)
10413 {
10414 // Reset the assembly handler to default
10416 // Set the new assembly handler. Note that the constructor actually
10417 // solves the original problem to get some initial conditions, but
10418 // this is OK because the RHS is always evaluated before assignment.
10420
10421 // If we are using a block solver, we must set the linear solver pointer
10422 // to the block hopf solver. The present linear solver is
10423 // used by the block solver and so must be passed as an argument.
10424 // The destructor of the Hopf handler returns the linear
10425 // solver to the original non-block version.
10426 if (block_solve)
10427 {
10429 }
10430 }
10431
10432
10433 //============================================================
10434 /// Activate the hopf tracking system by changing the assembly
10435 /// handler and initialising it using the parameter addressed
10436 /// by parameter_pt and the frequency and null vectors
10437 /// specified.
10438 //============================================================
10440 const double& omega,
10441 const DoubleVector& null_real,
10442 const DoubleVector& null_imag,
10443 const bool& block_solve)
10444 {
10445 // Reset the assembly handler to default
10447 // Set the new assembly handler. Note that the constructor actually
10448 // solves the original problem to get some initial conditions, but
10449 // this is OK because the RHS is always evaluated before assignment.
10451 new HopfHandler(this, parameter_pt, omega, null_real, null_imag);
10452
10453 // If we are using a block solver, we must set the linear solver pointer
10454 // to the block hopf solver. The present linear solver is
10455 // used by the block solver and so must be passed as an argument.
10456 // The destructor of the Hopf handler returns the linear
10457 // solver to the original non-block version.
10458 if (block_solve)
10459 {
10461 }
10462 }
10463
10464
10465 //===============================================================
10466 /// Reset the assembly handler to default
10467 //===============================================================
10469 {
10470 // If we have a non-default handler
10472 {
10473 // Delete the current assembly handler
10474 delete Assembly_handler_pt;
10475 // Reset the assembly handler
10477 }
10478 }
10479
10480 //===================================================================
10481 /// This function takes one step of length ds in pseudo-arclength.The
10482 /// argument parameter_pt is a pointer to the parameter (global variable)
10483 /// that is being traded for arc-length. The function returns the next desired
10484 /// arc-length according to criteria based upon the desired number of Newton
10485 /// Iterations per solve.
10486 //=====================================================================
10488 const double& ds,
10489 const unsigned& max_adapt)
10490 {
10491 // First check that we shouldn't use the other interface
10492 // by checking that the parameter isn't already stored as data
10494 {
10495 std::ostringstream error_message;
10496 error_message
10497 << "The parameter addressed by " << parameter_pt << " with the value "
10498 << *parameter_pt
10499 << "\n is supposed to be used for arc-length contiunation,\n"
10500 << " but it is stored in a Data object used by the problem.\n\n"
10501 << "This is bad for two reasons:\n"
10502 << "1. If it's a variable in the problem, it must already have an\n"
10503 "associated equation, so it can't be used for continuation;\n"
10504 << "2. The problem data will be reorganised in memory during "
10505 "continuation,\n"
10506 << " which means that the pointer will become invalid.\n\n"
10507 << "If you are sure that this is what you want to do you must:\n"
10508 << "A. Ensure that the value is pinned (don't worry we'll shout again "
10509 "if not)\n"
10510 << "B. Use the alternative interface\n"
10511 << " Problem::arc_length_step_solve(Data*,unsigned,...)\n"
10512 << " which uses a pointer to the data object and not the raw double "
10513 "pointer."
10514 << std::endl;
10515 throw OomphLibError(
10516 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
10517 }
10518
10519
10520 // If we are using the continuation timestepper
10522 {
10523 // Has the timestepper already been added to the problem
10525 const unsigned n_time_steppers = this->ntime_stepper();
10526 for (unsigned i = 0; i < n_time_steppers; i++)
10527 {
10529 {
10530 continuation_time_stepper_added = true;
10531 break;
10532 }
10533 }
10534
10535 // If not add it
10537 {
10538 oomph_info << "Adding the continuation time stepper\n";
10540 }
10541
10542 // Need to treat case of eigenproblems and bifurcation detection/tracking
10543 // here
10544
10545 // Backup the current timesteppers for each mesh!
10546
10547
10548 // If an arc length step has not been taken then set the timestepper
10550 {
10551 // Set the continuation timestepper for all data in the problem
10554 << " equation numbers allocated for continuation\n";
10555 }
10556
10557 } // End of continuation time stepper case
10558
10559
10560 // Just call the helper function (parameter is not from data)
10561 return arc_length_step_solve_helper(parameter_pt, ds, max_adapt);
10562 }
10563
10564
10565 //===================================================================
10566 /// This function takes one step of length ds in pseudo-arclength.The
10567 /// argument data_pt is a pointer to the data that holds the
10568 /// parameter (global variable)
10569 /// that is being traded for arc-length. The exact value is located at
10570 /// the location given by data_index.
10571 /// The function returns the next desired
10572 /// arc-length according to criteria based upon the desired number of Newton
10573 /// Iterations per solve.
10574 //=====================================================================
10576 const unsigned& data_index,
10577 const double& ds,
10578 const unsigned& max_adapt)
10579 {
10580 // Firstly check that the data is pinned
10581 if (!data_pt->is_pinned(data_index))
10582 {
10583 std::ostringstream error_stream;
10584 error_stream << "The value at index " << data_index
10585 << " in the data object to be used for continuation\n"
10586 << "is not pinned, which means that it is already a\n"
10587 << "variable in the problem "
10588 << "and cannot be used for continuation.\n\n"
10589 << "Please correct your formulation by either:\n"
10590 << "A. Pinning the value"
10591 << "\n or \n"
10592 << "B. Using a different parameter for continuation"
10593 << std::endl;
10594 throw OomphLibError(
10596 }
10597
10598
10599 // If we are using the continuation timestepper
10601 {
10602 // Has the timestepper already been added to the problem
10604 const unsigned n_time_steppers = this->ntime_stepper();
10605 for (unsigned i = 0; i < n_time_steppers; i++)
10606 {
10608 {
10609 continuation_time_stepper_added = true;
10610 break;
10611 }
10612 }
10613
10614 // If not add it
10616 {
10617 oomph_info << "Adding the continuation time stepper\n";
10619 }
10620
10621 // Need to treat case of eigenproblems and bifurcation detection/tracking
10622 // here
10623
10624
10625 // Backup the current timesteppers for each mesh!
10626
10627
10628 // If an arc length step has not been taken then set the timestepper
10630 {
10631 // Set the continuation timestepper for all data in the problem
10634 << " equation numbers allocated for continuation\n";
10635 }
10636
10637
10638 } // End of continuation time stepper case
10639
10640
10641 // Now make a pointer to the (newly allocated) data object
10642 double* parameter_pt = data_pt->value_pt(data_index);
10643 // Call the helper function, this will change the parameter_pt if
10644 // the data storage is changed (if the timestepper has to be changed,
10645 // which happens if this is the first time that a continuation step is
10646 // taken)
10647 // ALH: Don't think this is true because it has happened above....
10649 }
10650
10651 //======================================================================
10652 /// Private helper function that is used to set the appropriate
10653 /// pinned values for continuation. If the data is pinned, the its
10654 /// current value is always the same as the original value and
10655 /// the derivative is always zero. If these are not set properly
10656 /// then interpolation and projection in spatial adaptivity will
10657 /// not give the best answers.
10658 //=====================================================================
10660 {
10661 // Set the consistent values for the global mesh
10664
10665 // Deal with the spine meshes additional numbering separately
10666 const unsigned n_sub_mesh = this->nsub_mesh();
10667 // If there is only one mesh
10668 if (n_sub_mesh == 0)
10669 {
10670 if (SpineMesh* const spine_mesh_pt = dynamic_cast<SpineMesh*>(Mesh_pt))
10671 {
10672 spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10674 }
10675 // If it's a triangle mesh the we need to set the
10676 }
10677 // Otherwise loop over the sub meshes
10678 else
10679 {
10680 // Assign global equation numbers first
10681 for (unsigned i = 0; i < n_sub_mesh; i++)
10682 {
10683 if (SpineMesh* const spine_mesh_pt =
10684 dynamic_cast<SpineMesh*>(Sub_mesh_pt[i]))
10685 {
10686 spine_mesh_pt->set_consistent_pinned_spine_values_for_continuation(
10688 }
10689 }
10690 }
10691
10692 // Also set time stepper for global data
10693 const unsigned n_global = Global_data_pt.size();
10694 for (unsigned i = 0; i < n_global; ++i)
10695 {
10696 Continuation_time_stepper.set_consistent_pinned_values(Global_data_pt[i]);
10697 }
10698 }
10699
10700
10701 //===================================================================
10702 /// This function takes one step of length ds in pseudo-arclength.The
10703 /// argument parameter_pt is a pointer to the parameter (global variable)
10704 /// that is being traded for arc-length. The function returns the next desired
10705 /// arc-length according to criteria based upon the desired number of Newton
10706 /// Iterations per solve.
10707 //=====================================================================
10709 const double& ds,
10710 const unsigned& max_adapt)
10711 {
10712 //----------------------MAKE THE PROBLEM STEADY-----------------------
10713 // Loop over the timesteppers and make them (temporarily) steady.
10714 // We can only do continuation for steady problems!
10715 unsigned n_time_steppers = ntime_stepper();
10716 // Vector of bools to store the is_steady status of the various
10717 // timesteppers when we came in here
10718 std::vector<bool> was_steady(n_time_steppers);
10719
10720 // Loop over them all and make them (temporarily) static
10721 for (unsigned i = 0; i < n_time_steppers; i++)
10722 {
10725 }
10726
10727
10728 // Max number of solves
10729 unsigned max_solve = max_adapt + 1;
10730 // Storage for newton steps in each adaptation
10731 unsigned max_count_in_adapt_loop = 0;
10732
10733
10734 //----SET UP MEMORY FOR QUANTITIES THAT ARE REQUIRED OUTSIDE THE LOOP----
10735
10736 // Assign memory for solutions of the equations Jz = du/dparameter
10737 // This is needed here (outside the loop), so that we can save on
10738 // one linear solve when calculating the derivatives wrt the arc-length
10739 DoubleVector z;
10740
10741
10742 // Store sign of the Jacobian, used for bifurcation detection
10743 // If this is the first time that we are calling the arc-length solver,
10744 // this should not be used.
10746
10747 // Flag to indicate a sign change
10748 bool SIGN_CHANGE = false;
10749
10750
10751 // Adaptation loop
10752 for (unsigned isolve = 0; isolve < max_solve; ++isolve)
10753 {
10754 // Only adapt after the first solve has been done
10755 if (isolve > 0)
10756 {
10757 unsigned n_refined;
10758 unsigned n_unrefined;
10759
10760 // Adapt problem
10762
10763#ifdef OOMPH_HAS_MPI
10764 // Adaptation only converges if ALL the processes have no
10765 // refinement or unrefinement to perform
10766 unsigned total_refined = 0;
10767 unsigned total_unrefined = 0;
10769 {
10772 1,
10774 MPI_SUM,
10775 this->communicator_pt()->mpi_comm());
10779 1,
10781 MPI_SUM,
10782 this->communicator_pt()->mpi_comm());
10784 }
10785#endif
10786
10787 oomph_info << "---> " << n_refined << " elements were refined, and "
10788 << n_unrefined << " were unrefined"
10789#ifdef OOMPH_HAS_MPI
10790 << ", in total (over all processors).\n";
10791#else
10792 << ".\n";
10793#endif
10794
10795
10796 // Check convergence of adaptation cycle
10797 if ((n_refined == 0) && (n_unrefined == 0))
10798 {
10799 oomph_info << "\n \n Solution is fully converged in "
10800 << "Problem::newton_solver(). \n \n ";
10801 break;
10802 }
10803 }
10804
10805 //----------SAVE THE INITIAL VALUES, IN CASE THE STEP FAILS-----------
10806
10807 // Find the number of local dofs
10809
10810 // Only need to do this in the first loop
10811 if (isolve == 0)
10812 {
10814 {
10815 // Safety check, set up the array of dof derivatives, if necessary
10816 // The distribution is the same as the (natural) distribution of the
10817 // dofs
10818 if (Dof_derivative.size() != ndof_local)
10819 {
10820 Dof_derivative.resize(ndof_local, 0.0);
10821 }
10822
10823 // Safety check, set up the array of curren values, if necessary
10824 // Again the distribution reflects the (natural) distribution of the
10825 // dofs
10826 if (Dof_current.size() != ndof_local)
10827 {
10828 Dof_current.resize(ndof_local);
10829 }
10830 }
10831
10832 // Save the current value of the parameter
10834
10835 // Save the current values of the degrees of freedom
10836 for (unsigned long l = 0; l < ndof_local; l++)
10837 {
10838 dof_current(l) = *Dof_pt[l];
10839 }
10840
10841 // Set the value of ds_current
10842 Ds_current = ds;
10843 }
10844
10845 // Counter for the number of newton steps
10846 unsigned count = 0;
10847
10848 // Flag to indicate a successful step
10849 bool STEP_REJECTED = false;
10850
10851
10852 // Set the appropriate initial conditions for the pinned data
10854 {
10856 }
10857
10858 // Loop around the step in arc-length
10859 do
10860 {
10861 // Check that the step has not fallen below the minimum tolerance
10862 if (std::fabs(Ds_current) < Minimum_ds)
10863 {
10864 std::ostringstream error_message;
10865 error_message << "DESIRED ARC-LENGTH STEP " << Ds_current
10866 << " HAS FALLEN BELOW MINIMUM TOLERANCE, " << Minimum_ds
10867 << std::endl;
10868
10869 throw OomphLibError(error_message.str(),
10872 }
10873
10874 // Assume that we shall accept the step
10875 STEP_REJECTED = false;
10876
10877 // Set initial value of the parameter
10879
10880 // Perform any actions...
10882
10884
10885 // Loop over the (local) variables and set their initial values
10886 for (unsigned long l = 0; l < ndof_local; l++)
10887 {
10889 }
10890
10891 // Actually do the newton solve stage for the continuation problem
10892 try
10893 {
10895 }
10896 // Catch any exceptions thrown in the Newton solver
10897 catch (NewtonSolverError& error)
10898 {
10899 // Check whether it's the linear solver
10900 if (error.linear_solver_error)
10901 {
10902 std::ostringstream error_stream;
10903 error_stream << std::endl
10904 << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
10905 oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
10906 throw OomphLibError(error_stream.str(),
10909 }
10910 // Otherwise mark the step as having failed
10911 else
10912 {
10913 oomph_info << "STEP REJECTED --- TRYING AGAIN" << std::endl;
10914 STEP_REJECTED = true;
10915 // Let's take a smaller step
10916 Ds_current *= (2.0 / 3.0);
10917 }
10918 }
10919 } while (STEP_REJECTED); // continue until a step is accepted
10920
10921 // Set the maximum count
10923 {
10925 }
10926 } /// end of adaptation loop
10927
10928 // Only recalculate the derivatives if there has been a Newton solve
10929 // If not, the previous values should be close enough
10931 {
10932 //--------------------CHECK FOR POTENTIAL BIFURCATIONS-------------
10934 {
10935 // If the sign of the jacobian is zero issue a warning
10936 if (Sign_of_jacobian == 0)
10937 {
10938 std::string error_message =
10939 "The sign of the jacobian is zero after a linear solve\n";
10940 error_message += "Either the matrix is singular (unlikely),\n";
10941 error_message += "or the linear solver cannot compute the "
10942 "determinant of the matrix;\n";
10943 error_message += "e.g. an iterative linear solver.\n";
10944 error_message +=
10945 "If the latter, bifurcation detection must be via an eigensolver\n";
10946 OomphLibWarning(error_message,
10947 "Problem::arc_length_step_solve",
10949 }
10950 // If this is the first step, we cannot rely on the previous value
10951 // of the jacobian so set the previous sign to the present sign
10953 {
10955 }
10956 // If we have detected a sign change in the last converged Jacobian,
10957 // it must be a turning point or bifurcation
10959 {
10960 // There has been, at least, one sign change
10962
10963 // The sign has changed this time
10964 SIGN_CHANGE = true;
10965
10966 // Calculate the dot product of the approximate null vector
10967 // of the Jacobian matrix ((badly) approximated by z)
10968 // and the vectors of derivatives of the residuals wrt the
10969 // global parameter
10970 // If this is small it is a bifurcation rather than a turning point.
10971 // Get the derivative wrt global parameter
10972 // DoubleVector dparam;
10973 // get_derivative_wrt_global_parameter(parameter_pt,dparam);
10974 // Calculate the dot product
10975 // double dot=0.0;
10976 // for(unsigned long n=0;n<n_dofs;++n) {dot += dparam[n]*z[n];}
10977 // z.dot(dparam);
10978
10979 // Write the output message
10980 std::ostringstream message;
10981 message
10982 << "-----------------------------------------------------------";
10983 message << std::endl
10984 << "SIGN CHANGE IN DETERMINANT OF JACOBIAN: " << std::endl;
10985 message << "BIFURCATION OR TURNING POINT DETECTED BETWEEN "
10986 << Parameter_current << " AND " << *parameter_pt << std::endl;
10987 // message << "APPROXIMATE DOT PRODUCT : " << dot << "," << std::endl;
10988 // message << "IF CLOSE TO ZERO WE HAVE A BIFURCATION; ";
10989 // message << "OTHERWISE A TURNING POINT" << std::endl;
10990 message
10991 << "-----------------------------------------------------------"
10992 << std::endl;
10993
10994 // Write the message to standard output
10995 oomph_info << message.str();
10996
10997 // Open the information file for appending
10998 std::ofstream bifurcation_info("bifurcation_info",
10999 std::ios_base::app);
11000 // Write the message to the file
11001 bifurcation_info << message.str();
11002 bifurcation_info.close();
11003 }
11004 }
11005
11006 // Calculate the derivatives required for the next stage of continuation
11007 // In this we pass the last value of z (i.e. approximation)
11009 {
11011 }
11012 // Or use finite differences
11013 else
11014 {
11016 }
11017
11018 // If it's the first step then the value of the next step should
11019 // be the change in parameter divided by the parameter derivative
11020 // to obtain approximately the same parameter change
11022 {
11024 }
11025
11026 // We have taken our first step
11027 Arc_length_step_taken = true;
11028 }
11029 // If there has not been a newton step then we still need to estimate
11030 // the derivatives in the arc length direction
11031 else
11032 {
11033 // Default is to calculate the continuation derivatives by solving the
11034 // linear system. We must do this to ensure that the derivatives are in
11035 // sync It could lead to problems near turning points when we should
11036 // really be solving an eigenproblem, but seems OK so far!
11037
11038 // Save the current sign of the jacobian
11040
11041 // Calculate the continuation derivatives, which includes a solve
11042 // of the linear system if not using finite differences
11044 {
11046 }
11047 // Otherwise use finite differences
11048 else
11049 {
11051 }
11052
11053 // Reset the sign of the jacobian, just in case the sign has changed when
11054 // solving the continuation derivatives. The sign change will be picked
11055 // up on the next continuation step.
11057 }
11058
11059 // Reset the is_steady status of all timesteppers that
11060 // weren't already steady when we came in here and reset their
11061 // weights
11062 for (unsigned i = 0; i < n_time_steppers; i++)
11063 {
11064 if (!was_steady[i])
11065 {
11067 }
11068 }
11069
11070 // If we are trying to find a bifurcation and the first sign change
11071 // has occured, use bisection
11074 {
11075 // If there has been a sign change we need to half the step size
11076 // and reverse the direction
11077 if (SIGN_CHANGE)
11078 {
11079 Ds_current *= -0.5;
11080 }
11081 // Otherwise
11082 else
11083 {
11084 // The size of the bracketed interval is always
11085 // 2ds - Ds_current (this will work even if the original step failed)
11086 // We want our new step size to be half this
11087 Ds_current = ds - 0.5 * Ds_current;
11088 }
11089 // Return the desired value of the step
11090 return Ds_current;
11091 }
11092
11093 // If fewer than the desired number of Newton Iterations, increase the step
11095 {
11096 return Ds_current * 1.5;
11097 }
11098 // If more than the desired number of Newton Iterations, reduce the step
11100 {
11101 return Ds_current * (2.0 / 3.0);
11102 }
11103 // Otherwise return the step just taken
11104 return Ds_current;
11105 }
11106
11107
11108 //=======================================================================
11109 /// Take an explicit timestep of size dt
11110 //======================================================================
11111 void Problem::explicit_timestep(const double& dt, const bool& shift_values)
11112 {
11113#ifdef PARANOID
11114 if (this->explicit_time_stepper_pt() == 0)
11115 {
11116 throw OomphLibError("Explicit time stepper pointer is null in problem.",
11119 }
11120#endif
11121
11122 // Firstly we shift the time values
11123 if (shift_values)
11124 {
11126 }
11127 // Set the current value of dt, if we can
11128 if (time_pt()->ndt() > 0)
11129 {
11130 time_pt()->dt() = dt;
11131 }
11132
11133 // Take the explicit step
11134 this->explicit_time_stepper_pt()->timestep(this, dt);
11135 }
11136
11137
11138 //========================================================================
11139 /// Do one timestep of size dt using Newton's method with the specified
11140 /// tolerance and linear solver defined as member data of the Problem class.
11141 /// This will be the most commonly used version
11142 /// of unsteady_newton_solve, in which the time values are always shifted
11143 /// This does not include any kind of adaptativity. If the solution fails to
11144 /// converge the program will end.
11145 //========================================================================
11146 void Problem::unsteady_newton_solve(const double& dt)
11147 {
11148 // We shift the values, so shift_values is true
11149 unsteady_newton_solve(dt, true);
11150 }
11151
11152 //========================================================================
11153 /// Do one timestep forward of size dt using Newton's method with the
11154 /// specified tolerance and linear solver defined via member data of the
11155 /// Problem class.
11156 /// The boolean flag shift_values is used to control whether the time values
11157 /// should be shifted or not.
11158 //========================================================================
11159 void Problem::unsteady_newton_solve(const double& dt,
11160 const bool& shift_values)
11161 {
11162 // Shift the time values and the dts, according to the control flag
11163 if (shift_values)
11164 {
11166 }
11167
11168 // Advance global time and set current value of dt
11169 time_pt()->time() += dt;
11170 time_pt()->dt() = dt;
11171
11172 // Find out how many timesteppers there are
11173 unsigned n_time_steppers = ntime_stepper();
11174
11175 // Loop over them all and set the weights
11176 for (unsigned i = 0; i < n_time_steppers; i++)
11177 {
11179 }
11180
11181 // Run the individual timesteppers actions before timestep. These need to
11182 // be before the problem's actions_before_implicit_timestep so that the
11183 // boundary conditions are set consistently.
11184 for (unsigned i = 0; i < n_time_steppers; i++)
11185 {
11187 }
11188
11189 // Now update anything that needs updating before the timestep
11190 // This could be time-dependent boundary conditions, for example.
11192
11193 try
11194 {
11195 // Solve the non-linear problem for this timestep with Newton's method
11196 newton_solve();
11197 }
11198 // Catch any exceptions thrown in the Newton solver
11199 catch (NewtonSolverError& error)
11200 {
11201 oomph_info << std::endl
11202 << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
11203 // Check whether it's the linear solver
11204 if (error.linear_solver_error)
11205 {
11206 oomph_info << "ERROR IN THE LINEAR SOLVER" << std::endl;
11207 }
11208 // Check to see whether we have reached Max_iterations
11209 else if (error.iterations == Max_newton_iterations)
11210 {
11211 oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
11212 << ") REACHED WITHOUT CONVERGENCE " << std::endl;
11213 }
11214 // If not, it must be that we have exceeded the maximum residuals
11215 else
11216 {
11217 oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
11218 << " EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
11219 << std::endl;
11220 }
11221 // Die horribly!!
11222 std::ostringstream error_stream;
11223 error_stream << "Error occured in unsteady Newton solver. " << std::endl;
11224 throw OomphLibError(
11226 }
11227
11228 // Run the individual timesteppers actions, these need to be before the
11229 // problem's actions_after_implicit_timestep so that the time step is
11230 // finished before the problem does any auxiliary calculations (e.g. in
11231 // semi-implicit micromagnetics the calculation of magnetostatic field).
11232 for (unsigned i = 0; i < n_time_steppers; i++)
11233 {
11235 }
11236
11237
11238 // Now update anything that needs updating after the timestep
11241 }
11242
11243 //=======================================================================
11244 /// Attempt to take one timestep forward using dt_desired. The error control
11245 /// parameter, epsilon, is used to specify the desired approximate value of
11246 /// the global error norm per timestep. The routine returns the value an
11247 /// estimate of the next value of dt that should be taken.
11248 //=======================================================================
11250 const double& epsilon)
11251 {
11252 // We always want to shift the time values
11253 return adaptive_unsteady_newton_solve(dt_desired, epsilon, true);
11254 }
11255
11256
11257 //=======================================================================
11258 /// Attempt to take one timestep forward using the dt_desired.
11259 /// This is the driver for a number of adaptive solvers. If the solution
11260 /// fails to converge at a given timestep, the routine will automatically
11261 /// halve the time step and try again, until the time step falls below the
11262 /// specified minimum value. The routine returns the value an estimate
11263 /// of the next value of dt that should be taken.
11264 /// Timestep is also rejected if the error estimate post-solve
11265 /// (computed by global_temporal_error_norm()) exceeds epsilon.
11266 /// This behaviour can be over-ruled by setting the protected
11267 /// boolean Problem::Keep_temporal_error_below_tolerance to false.
11268 //========================================================================
11270 const double& epsilon,
11271 const bool& shift_values)
11272 {
11273 // First, we need to backup the existing dofs, in case the timestep is
11274 // rejected
11275
11276 // Find total number of dofs on current processor
11278
11279 // Now set up a Vector to hold current values
11281
11282 // Load values into dofs_current
11283 for (unsigned i = 0; i < n_dof_local; i++) dofs_current[i] = dof(i);
11284
11285 // Store the time
11286 double time_current = time_pt()->time();
11287
11288 // Flag to detect whether the timestep has been rejected or not
11289 bool reject_timestep = 0;
11290
11291 // Flag to detect whether any of the timesteppers are adaptive
11292 unsigned adaptive_flag = 0;
11293
11294 // The value of the actual timestep, by default the same as desired timestep
11295 double dt_actual = dt_desired;
11296
11297 // Find out whether any of the timesteppers are adaptive
11298 unsigned n_time_steppers = ntime_stepper();
11299 for (unsigned i = 0; i < n_time_steppers; i++)
11300 {
11301 if (time_stepper_pt(i)->adaptive_flag())
11302 {
11303 adaptive_flag = 1;
11304 break;
11305 }
11306 }
11307
11308 // Shift the time_values according to the control flag
11309 if (shift_values)
11310 {
11312 }
11313
11314 // This loop surrounds the adaptive time-stepping and will not be broken
11315 // until a timestep is accepted
11316 do
11317 {
11318 // Initially we assume that this step will succeed and that this dt
11319 // value is ok.
11320 reject_timestep = 0;
11321 double dt_rescaling_factor = 1.0;
11322
11323 // Set the new time and value of dt
11324 time_pt()->time() += dt_actual;
11325 time_pt()->dt() = dt_actual;
11326
11327 // Loop over all timesteppers and set the weights and predictor weights
11328 for (unsigned i = 0; i < n_time_steppers; i++)
11329 {
11330 // If the time_stepper is non-adaptive, this will be zero
11333 }
11334
11335 // Now calculate the predicted values for the all data and all positions
11337
11338 // Run the individual timesteppers actions before timestep. These need to
11339 // be before the problem's actions_before_implicit_timestep so that the
11340 // boundary conditions are set consistently.
11341 for (unsigned i = 0; i < n_time_steppers; i++)
11342 {
11344 }
11345
11346 // Do any updates/boundary conditions changes here
11348
11349 // Attempt to solve the non-linear system
11350 try
11351 {
11352 // Solve the non-linear problem at this timestep
11353 newton_solve();
11354 }
11355 // Catch any exceptions thrown
11356 catch (NewtonSolverError& error)
11357 {
11358 // If it's a solver error then die
11359 if (error.linear_solver_error ||
11361 {
11362 std::string error_message = "USER-DEFINED ERROR IN NEWTON SOLVER\n";
11363 error_message += "ERROR IN THE LINEAR SOLVER\n";
11364
11365 // Die
11366 throw OomphLibError(
11368 }
11369 else
11370 {
11371 // Reject the timestep, if we have an exception
11372 oomph_info << "TIMESTEP REJECTED" << std::endl;
11373 reject_timestep = 1;
11374
11375 // Half the time step
11377 }
11378 }
11379
11380 // Run the individual timesteppers actions, these need to be before the
11381 // problem's actions_after_implicit_timestep so that the time step is
11382 // finished before the problem does any auxiliary calculations (e.g. in
11383 // semi-implicit micromagnetics the calculation of magnetostatic field).
11384 for (unsigned i = 0; i < n_time_steppers; i++)
11385 {
11387 }
11388
11389 // Update anything that needs updating after the timestep
11391
11392 // If we have an adapative timestepper (and we haven't already failed)
11393 // then calculate the error estimate and rescaling factor.
11394 if (adaptive_flag && !reject_timestep)
11395 {
11396 // Once timestep has been accepted can do fancy error processing
11397 // Set the error weights
11398 for (unsigned i = 0; i < n_time_steppers; i++)
11399 {
11401 }
11402
11403 // Get a global error norm to use in adaptivity (as specified by the
11404 // problem sub-class writer). Prevent a divide by zero if the solution
11405 // gives very close to zero error. Error norm should never be negative
11406 // but use absolute value just in case.
11407 double error = std::max(std::abs(global_temporal_error_norm()), 1e-12);
11408
11409 // Calculate the scaling factor
11410 dt_rescaling_factor = std::pow(
11411 (epsilon / error), (1.0 / (1.0 + time_stepper_pt()->order())));
11412
11413 oomph_info << "Timestep scaling factor is " << dt_rescaling_factor
11414 << std::endl;
11415 oomph_info << "Estimated timestepping error is " << error << std::endl;
11416
11417
11418 // Do we have to do it again?
11419 if (error > epsilon)
11420 {
11421 oomph_info << "Estimated timestepping error " << error
11422 << " exceeds tolerance " << epsilon << " ";
11424 {
11425 oomph_info << " --> rejecting timestep." << std::endl;
11426 reject_timestep = 1;
11427 }
11428 else
11429 {
11430 oomph_info << " ...but we're not rejecting the timestep"
11431 << std::endl;
11432 }
11434 << "Note: This behaviour can be adjusted by changing the protected "
11435 << "boolean" << std::endl
11436 << std::endl
11437 << " Problem::Keep_temporal_error_below_tolerance" << std::endl;
11438 }
11439
11440
11441 } // End of if adaptive flag
11442
11443
11444 // Calculate the next time step size and check it's ok
11445 // ============================================================
11446
11447 // Calculate the possible next time step, if no error conditions
11448 // trigger.
11450
11451 // Check that the scaling factor is within the allowed range
11453 {
11454 oomph_info << "Tried to increase dt by the ratio "
11455 << dt_rescaling_factor << " which is above the maximum ("
11457 << "). Attempting to increase by the maximum ratio instead."
11458 << std::endl;
11460 }
11461 // If we have already rejected the timestep then don't do this check
11462 // because DTSF will definitely be too small.
11464 {
11465 // Handle this special case where we want to continue anyway (usually
11466 // Minimum_dt_but_still_proceed = -1 so this has no effect).
11468 {
11470 << "Warning: Adaptation of timestep to ensure satisfaction\n"
11471 << " of error bounds during adaptive timestepping\n"
11472 << " would lower dt below \n"
11473 << " Problem::Minimum_dt_but_still_proceed="
11475 << " ---> We're continuing with present timestep.\n"
11476 << std::endl;
11477 dt_rescaling_factor = 1.0;
11478 // ??ds shouldn't we set new_dt_candidate =
11479 // Minimum_dt_but_still_proceed here, rather than not changing dt at
11480 // all?
11481 }
11482 else
11483 {
11484 // Otherwise reject
11485 oomph_info << "Timestep would decrease by " << dt_rescaling_factor
11486 << " which is less than the minimum scaling factor "
11487 << DTSF_min_decrease << std::endl;
11488 oomph_info << "TIMESTEP REJECTED" << std::endl;
11489 reject_timestep = 1;
11490 }
11491 }
11492
11493 // Now check that the new dt is within the allowed range
11495 {
11496 oomph_info << "Tried to increase dt to " << new_dt_candidate
11497 << " which is above the maximum (" << Maximum_dt
11498 << "). I increased it to the maximum value instead.";
11500 }
11501 else if (new_dt_candidate < Minimum_dt)
11502 {
11503 std::ostringstream err;
11504 err << "Tried to reduce dt to " << new_dt_candidate
11505 << " which is less than the minimum dt (" << Minimum_dt << ")."
11506 << std::endl;
11507 throw OomphLibError(
11509 }
11510 else
11511 {
11513 }
11514
11515
11517
11518
11519 // If we are rejecting this attempt then revert the dofs etc.
11520 if (reject_timestep)
11521 {
11522 // Reset the time
11523 time_pt()->time() = time_current;
11524
11525 // Reload the dofs
11526 unsigned ni = dofs_current.size();
11527 for (unsigned i = 0; i < ni; i++)
11528 {
11529 dof(i) = dofs_current[i];
11530 }
11531
11532#ifdef OOMPH_HAS_MPI
11533 // Synchronise the solution on different processors (on each submesh)
11534 this->synchronise_all_dofs();
11535#endif
11536
11537 // Call all "after" actions, e.g. to handle mesh updates
11543 }
11544
11545 }
11546 // Keep this loop going until we accept the timestep
11547 while (reject_timestep);
11548
11549 // Once the timestep has been accepted, return the time step that should be
11550 // used next time.
11551 return dt_actual;
11552 }
11553
11554
11555 //=======================================================================
11556 /// Private helper function to perform
11557 /// unsteady "doubly" adaptive Newton solve: Does temporal
11558 /// adaptation first, i.e. we try to do a timestep with an increment
11559 /// of dt, and adjusting dt until the solution on the given mesh satisfies
11560 /// the temporal error measure with tolerance epsilon. Following
11561 /// this, we do up to max_adapt spatial adaptions (without
11562 /// re-examining the temporal error). If first==true, the initial conditions
11563 /// are re-assigned after the mesh adaptations.
11564 /// Shifting of time can be suppressed by overwriting the
11565 /// default value of shift (true). [Shifting must be done
11566 /// if first_timestep==true because we're constantly re-assigning
11567 /// the initial conditions; if first_timestep==true and shift==false
11568 /// shifting is performed anyway and a warning is issued.
11569 /// Pseudo-Boolean flag suppress_resolve_after_spatial_adapt [0: false;
11570 /// 1: true] does what it says.]
11571 //========================================================================
11573 const double& dt_desired,
11574 const double& epsilon,
11575 const unsigned& max_adapt,
11577 const bool& first,
11578 const bool& shift_values)
11579 {
11580 // Store the initial time
11581 double initial_time = time_pt()->time();
11582
11583 // Take adaptive timestep, adjusting dt until tolerance is satisfied
11584 double new_dt =
11586 double dt_taken = time_pt()->dt();
11587 oomph_info << "Accepted solution taken with timestep: " << dt_taken
11588 << std::endl;
11589
11590
11591 // Bail out straightaway if no spatial adaptation allowed
11592 if (max_adapt == 0)
11593 {
11594 oomph_info << "No spatial refinement allowed; max_adapt=0\n";
11595 return new_dt;
11596 }
11597
11598 // Adapt problem/mesh
11599 unsigned n_refined = 0;
11600 unsigned n_unrefined = 0;
11602
11603 // Check if mesh has been adapted on other processors
11607
11608
11609#ifdef OOMPH_HAS_MPI
11611 {
11612 // Sum n_refine across all processors
11614 ref_count[0] = n_refined;
11617 &total_ref_count[0],
11618 2,
11619 MPI_INT,
11620 MPI_SUM,
11622 }
11623#endif
11624
11625
11626 // Re-solve the problem if the adaptation has changed anything
11627 if ((total_ref_count[0] != 0) || (total_ref_count[1] != 0))
11628 {
11630 {
11631 oomph_info << "Mesh was adapted but re-solve has been suppressed."
11632 << std::endl;
11633 }
11634 else
11635 {
11637 << "Mesh was adapted --> we'll re-solve for current timestep."
11638 << std::endl;
11639
11640 // Reset time to what it was when we entered here
11641 // because it will be incremented again by dt_taken.
11642 time_pt()->time() = initial_time;
11643
11644 // Shift the timesteps? No! They've been shifted already when we
11645 // called the solve with pure temporal adaptivity...
11646 bool shift = false;
11647
11648 // Reset the inital condition on refined meshes
11649 if (first)
11650 {
11651 // Reset default set_initial_condition has been called flag to false
11653
11654 // Reset the initial conditions
11655 oomph_info << "Re-assigning initial condition at time="
11656 << time_pt()->time() << std::endl;
11658
11659 // This is the first timestep so shifting
11660 // has to be done following the assignment of initial conditions,
11661 // providing the default set_initial_condition function has not
11662 // been called.
11663 // In fact, unsteady_newton_solve(...) does that automatically.
11664 // We're changing the flag here to avoid warning messages.
11666 {
11667 shift = true;
11668 }
11669 }
11670
11671 // Now take the step again on the refined mesh, using the same
11672 // timestep as used before.
11674 }
11675 }
11676 else
11677 {
11678 oomph_info << "Mesh wasn't adapted --> we'll accept spatial refinement."
11679 << std::endl;
11680 }
11681
11682 return new_dt;
11683 }
11684
11685
11686 //========================================================================
11687 /// Initialise the previous values of the variables for time stepping
11688 /// corresponding to an impulsive start. Previous history for all data
11689 /// is generated by the appropriate timesteppers. Previous nodal
11690 /// positions are simply copied backwards.
11691 //========================================================================
11693 {
11694 // Assign the impulsive values in the "master" mesh
11696
11697 // Loop over global data
11698 unsigned Nglobal = Global_data_pt.size();
11699 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11700 {
11702 ->time_stepper_pt()
11704 }
11705 }
11706
11707
11708 //=======================================================================
11709 /// Assign the values for an impulsive start and also set the initial
11710 /// values of the previous dts to both be dt
11711 //======================================================================
11713 {
11714 // First initialise the dts and set the weights
11715 initialise_dt(dt);
11716 // Now call assign_initial_values_impulsive
11718 }
11719
11720 //=======================================================================
11721 /// Return the current value of continuous time. If not Time object
11722 /// has been assigned, then throw an error
11723 //======================================================================
11725 {
11726 if (Time_pt == 0)
11727 {
11728 throw OomphLibError("Time object has not been set",
11731 }
11732 else
11733 {
11734 return Time_pt->time();
11735 }
11736 }
11737
11738 //=======================================================================
11739 /// Return the current value of continuous time. If not Time object
11740 /// has been assigned, then throw an error. Const version.
11741 //======================================================================
11742 double Problem::time() const
11743 {
11744 if (Time_pt == 0)
11745 {
11746 throw OomphLibError("Time object has not been set",
11749 }
11750 else
11751 {
11752 return Time_pt->time();
11753 }
11754 }
11755
11756
11757 //=======================================================================
11758 /// Set all problem data to have the same timestepper (timestepper_pt).
11759 /// This is mainly used in continuation and bifurcation detection problems
11760 /// in which case the total number of unknowns may change and the changes
11761 /// to the underlying memory layout means that the Dof_pt must be
11762 /// reallocated. Thus, the function calls assign_eqn_numbers() and returns
11763 /// the number of new equation numbers.
11764 //=========================================================================
11766 TimeStepper* const& time_stepper_pt, const bool& preserve_existing_data)
11767 {
11768 // Set the timestepper for the master mesh's nodal and elemental data
11769 // to be the
11770 // continuation time stepper. This will wipe all storage other than
11771 // the 0th (present time) value at all the data objects
11774
11775 // Deal with the any additional mesh level timestepper data separately
11776 const unsigned n_sub_mesh = this->nsub_mesh();
11777 // If there is only one mesh
11778 if (n_sub_mesh == 0)
11779 {
11782 }
11783 // Otherwise loop over the sub meshes
11784 else
11785 {
11786 // Assign global equation numbers first
11787 for (unsigned i = 0; i < n_sub_mesh; i++)
11788 {
11789 this->Sub_mesh_pt[i]->set_mesh_level_time_stepper(
11790 time_stepper_pt, preserve_existing_data);
11791 }
11792 }
11793
11794 // Also set time stepper for global data
11795 const unsigned n_global = Global_data_pt.size();
11796 for (unsigned i = 0; i < n_global; ++i)
11797 {
11798 Global_data_pt[i]->set_time_stepper(time_stepper_pt,
11800 }
11801
11802 // We now need to reassign equations numbers because the Dof pointer
11803 // will be inappropriate because memory has been reallocated
11804
11805#ifdef OOMPH_HAS_MPI
11807 {
11808 std::ostringstream warning_stream;
11809 warning_stream << "This has not been comprehensively tested for "
11810 "distributed problems.\n"
11811 << "I'm sure that I need to worry about external halo and "
11812 "external elements."
11813 << std::endl;
11816 }
11817
11818#endif
11819
11820 return (this->assign_eqn_numbers());
11821 }
11822
11823
11824 //========================================================================
11825 /// Shift all time-dependent data along for next timestep.
11826 //========================================================================
11828 {
11829 // Move the values of dt in the Time object
11830 Time_pt->shift_dt();
11831
11832 // Only shift time values in the "master" mesh, otherwise things will
11833 // get shifted twice in complex problems
11835
11836 // Shift global data with their own timesteppers
11837 unsigned Nglobal = Global_data_pt.size();
11838 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11839 {
11842 }
11843 }
11844
11845
11846 //========================================================================
11847 /// Calculate the predictions of all variables in problem
11848 //========================================================================
11850 {
11851// Check that if we have multiple time steppers none of them want to
11852// predict by calling an explicit timestepper (as opposed to doing
11853// something like an explicit step by combining known history values, as
11854// done in BDF).
11855#ifdef PARANOID
11856 if (Time_stepper_pt.size() != 1)
11857 {
11858 for (unsigned j = 0; j < Time_stepper_pt.size(); j++)
11859 {
11860 if (time_stepper_pt()->predict_by_explicit_step())
11861 {
11862 std::string err = "Prediction by explicit step only works for "
11863 "problems with a simple time";
11864 err += "stepper. I think implementing anything more general will";
11865 err += "require a rewrite of explicit time steppers. - David";
11866 throw OomphLibError(
11868 }
11869 }
11870 }
11871#endif
11872
11873
11874 // Predict using an explicit timestepper (don't do it if adaptive = false
11875 // because pointers probably aren't set up).
11876 if (time_stepper_pt()->predict_by_explicit_step() &&
11877 time_stepper_pt()->adaptive_flag())
11878 {
11879 // Copy the time stepper's predictor pt into problem's explicit time
11880 // stepper pt (unless problem already has its own explicit time
11881 // stepper).
11883#ifdef PARANOID
11884 if (ets_pt == 0)
11885 {
11886 std::string err = "Requested predictions by explicit step but explicit";
11887 err += " predictor pt is null.";
11888 throw OomphLibError(
11890 }
11891
11892 if ((explicit_time_stepper_pt() != ets_pt) &&
11893 (explicit_time_stepper_pt() != 0))
11894 {
11895 throw OomphLibError("Problem has explicit time stepper other than "
11896 "predictor, not sure how to handle this yet ??ds",
11899 }
11900#endif
11902
11903 // Backup dofs and time
11905
11906#ifdef PARANOID
11907 double backup_time = time();
11908#endif
11909
11910 // Move time back so that we are at the start of the timestep (as
11911 // explicit_timestep functions expect). This is needed because the
11912 // predictor calculations are done after unsteady newton solve has
11913 // started, and it has already moved time forwards.
11914 double dt = time_pt()->dt();
11915 time() -= dt;
11916
11917 // Explicit step
11918 this->explicit_timestep(dt, false);
11919
11920 // Copy predicted dofs and time to their storage slots.
11921 set_dofs(time_stepper_pt()->predictor_storage_index(), Dof_pt);
11923
11924 // Check we got the times right
11925#ifdef PARANOID
11926 if (std::abs(time() - backup_time) > 1e-12)
11927 {
11928 using namespace StringConversion;
11929 std::string err = "Predictor landed at the wrong time!";
11930 err += " Expected time " + to_string(backup_time, 14) + " but got ";
11931 err += to_string(time(), 14);
11932 throw OomphLibError(
11934 }
11935#endif
11936
11937 // Restore dofs and time
11939 }
11940
11941 // Otherwise we can do predictions in a more object oriented way using
11942 // whatever timestepper the data provides (this is the normal case).
11943 else
11944 {
11945 // Calculate all predictions in the "master" mesh
11947
11948 // Calculate predictions for global data with their own timesteppers
11949 unsigned Nglobal = Global_data_pt.size();
11950 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
11951 {
11954 }
11955 }
11956
11957 // If requested then copy the predicted value into the current time data
11958 // slots, ready for the newton solver to use as an initial guess.
11960 {
11961 // Not sure I know enough about distributed problems to implement
11962 // this. Probably you just need to loop over ndof_local or something,
11963 // but I can't really test it...
11964#ifdef OOMPH_HAS_MPI
11965 if (distributed())
11966 {
11967 throw OomphLibError("Not yet implemented for distributed problems",
11970 }
11971#endif
11972
11973 // With multiple time steppers this is much more complex becuase you
11974 // need to check the time stepper for each data to get the
11975 // predictor_storage_index(). Do-able if you need it though.
11976 if (Time_stepper_pt.size() != 1)
11977 {
11978 std::string err = "Not implemented for multiple time steppers";
11979 throw OomphLibError(
11981 }
11982
11983 // Get predicted values
11985 get_dofs(time_stepper_pt()->predictor_storage_index(), predicted_dofs);
11986
11987 // Update dofs at current step
11988 for (unsigned i = 0; i < ndof(); i++)
11989 {
11990 dof(i) = predicted_dofs[i];
11991 }
11992 }
11993 }
11994
11995 //======================================================================
11996 /// Enable recycling of the mass matrix in explicit timestepping
11997 /// schemes. Useful for timestepping on fixed meshes when you want
11998 /// to avoid the linear solve phase.
11999 //=====================================================================
12001 {
12004
12005 // If we have a discontinuous formulation set the elements to reuse
12006 // their own mass matrices
12008 {
12009 const unsigned n_element = Problem::mesh_pt()->nelement();
12010 // Loop over the other elements
12011 for (unsigned e = 0; e < n_element; e++)
12012 {
12013 // Cache the element
12014 DGElement* const elem_pt =
12015 dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
12016 elem_pt->enable_mass_matrix_reuse();
12017 }
12018 }
12019 }
12020
12021 //======================================================================
12022 /// Turn off the recyling of the mass matrix in explicit
12023 /// time-stepping schemes
12024 //======================================================================
12026 {
12029
12030 // If we have a discontinuous formulation set the element-level
12031 // function
12033 {
12034 const unsigned n_element = Problem::mesh_pt()->nelement();
12035 // Loop over the other elements
12036 for (unsigned e = 0; e < n_element; e++)
12037 {
12038 // Cache the element
12039 DGElement* const elem_pt =
12040 dynamic_cast<DGElement*>(Problem::mesh_pt()->element_pt(e));
12041 elem_pt->disable_mass_matrix_reuse();
12042 }
12043 }
12044 }
12045
12046
12047 //=========================================================================
12048 /// Copy Data values, nodal positions etc from specified problem.
12049 /// Note: This is not a copy constructor. We assume that the current
12050 /// and the "original" problem have both been created by calling
12051 /// the same problem constructor so that all Data objects,
12052 /// time steppers etc. in the two problems are completely independent.
12053 /// This function copies the nodal, internal and global values
12054 /// and the time parameters from the original problem into "this"
12055 /// one. This functionality is required, e.g. for
12056 /// multigrid computations.
12057 //=========================================================================
12059 {
12060 // Copy time
12061 //----------
12062
12063 // Flag to indicate that orig problem is unsteady problem
12064 bool unsteady_flag = (orig_problem_pt->time_pt() != 0);
12065
12066 // Copy current time and previous time increments for proper unsteady run
12067 if (unsteady_flag)
12068 {
12069 oomph_info << "Copying an unsteady problem." << std::endl;
12070 // Current time
12071 this->time_pt()->time() = orig_problem_pt->time_pt()->time();
12072 // Timesteps
12073 unsigned n_dt = orig_problem_pt->time_pt()->ndt();
12074 time_pt()->resize(n_dt);
12075 for (unsigned i = 0; i < n_dt; i++)
12076 {
12077 time_pt()->dt(i) = orig_problem_pt->time_pt()->dt(i);
12078 }
12079
12080 // Find out how many timesteppers there are
12081 unsigned n_time_steppers = ntime_stepper();
12082
12083 // Loop over them all and set the weights
12084 for (unsigned i = 0; i < n_time_steppers; i++)
12085 {
12087 }
12088 }
12089
12090 // Copy nodes
12091 //-----------
12092
12093 // Loop over submeshes:
12094 unsigned nmesh = nsub_mesh();
12095 if (nmesh == 0) nmesh = 1;
12096 for (unsigned m = 0; m < nmesh; m++)
12097 {
12098 // Find number of nodes in present mesh
12099 unsigned long n_node = mesh_pt(m)->nnode();
12100
12101 // Check # of nodes:
12102 unsigned long n_node_orig = orig_problem_pt->mesh_pt(m)->nnode();
12103 if (n_node != n_node_orig)
12104 {
12105 std::ostringstream error_message;
12106 error_message << "Number of nodes in copy " << n_node
12107 << " not equal to the number in the original "
12108 << n_node_orig << std::endl;
12109
12110 throw OomphLibError(error_message.str(),
12113 }
12114
12115 // Loop over the nodes
12116 for (unsigned long i = 0; i < n_node; i++)
12117 {
12118 // Try to cast to elastic node
12120 dynamic_cast<SolidNode*>(mesh_pt(m)->node_pt(i));
12121 if (el_node_pt != 0)
12122 {
12124 dynamic_cast<SolidNode*>(orig_problem_pt->mesh_pt(m)->node_pt(i));
12126 }
12127 else
12128 {
12129 mesh_pt(m)->node_pt(i)->copy(orig_problem_pt->mesh_pt(m)->node_pt(i));
12130 }
12131 }
12132 }
12133
12134
12135 // Copy global data:
12136 //------------------
12137
12138 // Number of global data
12139 unsigned n_global = Global_data_pt.size();
12140
12141 // Check # of nodes in orig problem
12142 unsigned long n_global_orig = orig_problem_pt->nglobal_data();
12143 if (n_global != n_global_orig)
12144 {
12145 std::ostringstream error_message;
12146 error_message << "Number of global data in copy " << n_global
12147 << " not equal to the number in the original "
12148 << n_global_orig << std::endl;
12149
12150 throw OomphLibError(
12151 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
12152 }
12153
12154 for (unsigned iglobal = 0; iglobal < n_global; iglobal++)
12155 {
12156 Global_data_pt[iglobal]->copy(orig_problem_pt->global_data_pt(iglobal));
12157 }
12158
12159
12160 // Copy internal data of elements:
12161 //--------------------------------
12162
12163 // Loop over submeshes:
12164 for (unsigned m = 0; m < nmesh; m++)
12165 {
12166 // Loop over elements and deal with internal data
12167 unsigned n_element = mesh_pt(m)->nelement();
12168 for (unsigned e = 0; e < n_element; e++)
12169 {
12171 unsigned n_internal = el_pt->ninternal_data();
12172 if (n_internal > 0)
12173 {
12174 // Check # of internals :
12175 unsigned long n_internal_orig =
12176 orig_problem_pt->mesh_pt(m)->element_pt(e)->ninternal_data();
12178 {
12179 std::ostringstream error_message;
12180 error_message << "Number of internal data in copy " << n_internal
12181 << " not equal to the number in the original "
12182 << n_internal_orig << std::endl;
12183
12184 throw OomphLibError(error_message.str(),
12187 }
12188 for (unsigned i = 0; i < n_internal; i++)
12189 {
12191 orig_problem_pt->mesh_pt(m)->element_pt(e)->internal_data_pt(i));
12192 }
12193 }
12194 }
12195 }
12196 }
12197
12198 //=========================================================================
12199 /// Make and return a pointer to the copy of the problem. A virtual
12200 /// function that must be filled in by the user is they wish to perform
12201 /// adaptive refinement in bifurcation tracking or in multigrid problems.
12202 /// ALH: WILL NOT BE NECESSARY IN BIFURCATION TRACKING IN LONG RUN...
12203 //=========================================================================
12205 {
12206 std::ostringstream error_stream;
12208 << "This function must be overloaded in your specific problem, and must\n"
12209 << "create an exact copy of your problem. Usually this will be achieved\n"
12210 << "by a call to the constructor with exactly the same arguments as "
12211 "used\n";
12212
12213 throw OomphLibError(
12215 }
12216
12217
12218 //=========================================================================
12219 /// Dump refinement pattern of all refineable meshes and all generic
12220 /// Problem data to file for restart.
12221 //=========================================================================
12222 void Problem::dump(std::ofstream& dump_file) const
12223 {
12224 // Number of submeshes?
12225 unsigned n_mesh = nsub_mesh();
12226
12227 dump_file << std::max(unsigned(1), n_mesh) << " # number of (sub)meshes "
12228 << std::endl;
12229
12230 // Single mesh:
12231 //------------
12232 if (n_mesh == 0)
12233 {
12234 // Dump level of refinement before pruning
12236 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12237 {
12238 dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12239 << " # uniform refinement when pruned " << std::endl;
12240 }
12241 else
12242 {
12243 dump_file << 0 << " # (fake) uniform refinement when pruned "
12244 << std::endl;
12245 }
12246 dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12247 }
12248
12249 // Multiple submeshes
12250 //------------------
12251 else
12252 {
12253 // Loop over submeshes to dump level of refinement before pruning
12254 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12255 {
12257 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12258 {
12259 dump_file << mmesh_pt->uniform_refinement_level_when_pruned()
12260 << " # uniform refinement when pruned " << std::endl;
12261 }
12262 else
12263 {
12264 dump_file << 0 << " # (fake) uniform refinement when pruned "
12265 << std::endl;
12266 }
12267 }
12268 dump_file << 9999 << " # test flag for end of sub-meshes " << std::endl;
12269 }
12270
12271#ifdef OOMPH_HAS_MPI
12272
12273 const int my_rank = this->communicator_pt()->my_rank();
12274
12275 // Record destination of all base elements
12276 unsigned n = Base_mesh_element_pt.size();
12279 for (unsigned e = 0; e < n; e++)
12280 {
12282 if (el_pt != 0)
12283 {
12284 if (!el_pt->is_halo())
12285 {
12287 }
12288 }
12289 }
12290
12291
12292 // Get target for all base elements by reduction
12294 {
12295 // Check that the base elements have been associated to a processor
12296 // (the Base_mesh_elemen_pt is only used for structured meshes,
12297 // therefore, if there are no ustructured meshes as part of the
12298 // problem this container will be empty)
12299 if (n > 0)
12300 {
12303 n,
12304 MPI_INT,
12305 MPI_MAX,
12306 this->communicator_pt()->mpi_comm());
12307 }
12308 }
12309 else
12310 {
12311 // All the same...
12313 }
12314
12315
12316 dump_file << n << " # Number of base elements; partitioning follows.\n";
12317 for (unsigned e = 0; e < n; e++)
12318 {
12320 }
12321 dump_file << "8888 #test flag for end of base element distribution\n";
12322
12323#endif
12324
12325 // Single mesh:
12326 //------------
12327 if (n_mesh == 0)
12328 {
12329 // Dump single mesh refinement pattern (if mesh is refineable)
12331 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12332 {
12333 mmesh_pt->dump_refinement(dump_file);
12334 }
12335#ifdef OOMPH_HAS_TRIANGLE_LIB
12336 // Dump triangle mesh TriangulateIO which represents mesh topology
12337 TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12338 if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12339 {
12340#ifdef OOMPH_HAS_MPI
12341 // Check if the mesh is distributed, if that is the case then
12342 // additional info. needs to be saved
12343 if (mmesh_pt->is_mesh_distributed())
12344 {
12345 // Dump the info. related with the distribution of the mesh
12346 mmesh_pt->dump_distributed_info_for_restart(dump_file);
12347 }
12348#endif
12349 mmesh_pt->dump_triangulateio(dump_file);
12350 }
12351#endif
12352 }
12353
12354 // Multiple submeshes
12355 //------------------
12356 else
12357 {
12358 // Loop over submeshes
12359 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
12360 {
12361 // Dump single mesh refinement pattern (if mesh is refineable)
12363 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
12364 {
12365 mmesh_pt->dump_refinement(dump_file);
12366 }
12367#ifdef OOMPH_HAS_TRIANGLE_LIB
12368 // Dump triangle mesh TriangulateIO which represents mesh topology
12370 dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
12371 if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12372 {
12373#ifdef OOMPH_HAS_MPI
12374 // Check if the mesh is distributed, if that is the case then
12375 // additional info. needs to be saved
12376 if (mmesh_pt->is_mesh_distributed())
12377 {
12378 // Dump the info. related with the distribution of the mesh
12379 mmesh_pt->dump_distributed_info_for_restart(dump_file);
12380 }
12381#endif
12382 mmesh_pt->dump_triangulateio(dump_file);
12383 }
12384#endif
12385 } // End of loop over submeshes
12386 }
12387
12388
12389 // Dump time
12390 // ---------
12391
12392 // Flag to indicate unsteady run
12393 bool unsteady_flag = (time_pt() != 0);
12394 dump_file << unsteady_flag << " # bool flag for unsteady" << std::endl;
12395
12396 // Current time and previous time increments for proper unsteady run
12397 if (unsteady_flag)
12398 {
12399 // Current time
12400 dump_file << time_pt()->time() << " # Time " << std::endl;
12401 // Timesteps
12402 unsigned n_dt = time_pt()->ndt();
12403 dump_file << n_dt << " # Number of timesteps " << std::endl;
12404 for (unsigned i = 0; i < n_dt; i++)
12405 {
12406 dump_file << time_pt()->dt(i) << " # dt " << std::endl;
12407 }
12408 }
12409 // Dummy time and previous time increments for steady run
12410 else
12411 {
12412 // Current time
12413 dump_file << "0.0 # Dummy time from steady run " << std::endl;
12414 // Timesteps
12415 dump_file << "0 # Dummy number of timesteps from steady run" << std::endl;
12416 }
12417
12418 // Loop over submeshes and dump their data
12419 unsigned nmesh = nsub_mesh();
12420 if (nmesh == 0) nmesh = 1;
12421 for (unsigned m = 0; m < nmesh; m++)
12422 {
12424 }
12425
12426 // Dump global data
12427
12428 // Loop over global data
12429 unsigned Nglobal = Global_data_pt.size();
12430 dump_file << Nglobal << " # number of global Data items " << std::endl;
12431 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
12432 {
12434 dump_file << std::endl;
12435 }
12436 }
12437
12438 //=========================================================================
12439 /// Read refinement pattern of all refineable meshes and refine them
12440 /// accordingly, then read all Data and nodal position info from
12441 /// file for restart. Return flag to indicate if the restart was from
12442 /// steady or unsteady solution.
12443 //=========================================================================
12444 void Problem::read(std::ifstream& restart_file, bool& unsteady_restart)
12445 {
12446 // Check if the file is actually open as it won't be if it doesn't
12447 // exist! In that case we're almost certainly restarting the run on
12448 // a larger number of processors than the restart data was produced.
12449 // Say so and return
12450 bool restart_file_is_open = true;
12451 if (!restart_file.is_open())
12452 {
12453 std::ostringstream warn_message;
12454 warn_message << "Restart file isn't open -- I'm assuming that this is\n";
12455 warn_message << "because we're restarting on a larger number of\n";
12456 warn_message << "processor than were in use when the restart data was \n";
12457 warn_message << "dumped.\n";
12459 warn_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12460 restart_file_is_open = false;
12461 }
12462
12463 // Number of (sub)meshes?
12464 unsigned n_mesh = std::max(unsigned(1), nsub_mesh());
12465
12466 std::string input_string;
12467
12468 // Read line up to termination sign
12470
12471 // Ignore rest of line
12472 restart_file.ignore(80, '\n');
12473
12474 // Read in number of sub-meshes
12475 unsigned n_submesh_read;
12476 n_submesh_read = std::atoi(input_string.c_str());
12477
12478#ifdef PARANOID
12480 {
12481 if (n_submesh_read != n_mesh)
12482 {
12483 std::ostringstream error_message;
12484 error_message
12485 << "Number of sub-meshes specified in restart file, "
12486 << n_submesh_read << " doesn't \n match the my number of sub-meshes,"
12487 << n_mesh << std::endl
12488 << "Make sure all sub-meshes have been added to the global mesh\n"
12489 << "when calling the Problem::dump() function.\n";
12490 throw OomphLibError(error_message.str(),
12493 }
12494 }
12495#else
12496 // Suppress comiler warnings about non-used variable
12499#endif
12500
12501
12502 // Read levels of refinement before pruning
12503#ifdef OOMPH_HAS_MPI
12504 bool refine_and_prune_required = false;
12505#endif
12507 for (unsigned i = 0; i < n_mesh; i++)
12508 {
12509 // Read line up to termination sign
12511
12512 // Ignore rest of line
12513 restart_file.ignore(80, '\n');
12514
12515 // Convert
12516 nrefinement_for_mesh[i] = std::atoi(input_string.c_str());
12517
12518 // Get pointer to sub-mesh in incarnation as tree-based refineable mesh
12520 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(i));
12521
12522 // If it's not a tree-based refineable mesh, ignore the following
12523 if (ref_mesh_pt == 0)
12524 {
12525 if (nrefinement_for_mesh[i] != 0)
12526 {
12527 std::ostringstream error_stream;
12528 error_stream << "Nonzero uniform-refinement-when-pruned specified\n"
12529 << "even though mesh is not tree-based. Odd. May want\n"
12530 << "to check this carefully before disabling this \n"
12531 << "warning/error -- most likely if/when we start to\n"
12532 << "prune unstructured meshes [though I can't see why\n"
12533 << "we would want to do this, given that they are \n"
12534 << "currently totally re-generated...]\n";
12535 throw OomphLibError(error_stream.str(),
12538 }
12539 }
12540 else
12541 {
12542 // Get min and max refinement level
12543 unsigned local_min_ref = 0;
12544 unsigned local_max_ref = 0;
12545 ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
12546
12547 // Overall min refinement level over all meshes
12548 unsigned min_ref = local_min_ref;
12549
12550#ifdef OOMPH_HAS_MPI
12552 {
12553 // Reconcile between processors: If (e.g. following
12554 // distribution/pruning) the mesh has no elements on this
12555 // processor) then ignore its contribution to the poll of
12556 // max/min refinement levels
12558 if (ref_mesh_pt->nelement() == 0)
12559 {
12561 }
12562 int int_min_ref = 0;
12564 &int_min_ref,
12565 1,
12566 MPI_INT,
12567 MPI_MIN,
12568 Communicator_pt->mpi_comm());
12569
12570 // Overall min refinement level over all meshes
12572 }
12573#endif
12574
12575 // Need to refine less
12577 {
12579 }
12580 }
12581
12582#ifdef OOMPH_HAS_MPI
12583 if (nrefinement_for_mesh[i] > 0)
12584 {
12586 }
12587#endif
12588 }
12589
12590
12591 // Reconcile overall need to refine and prune (even empty
12592 // processors have to participate in some communication!)
12593#ifdef OOMPH_HAS_MPI
12595 {
12596 unsigned local_req_flag = 0;
12597 unsigned req_flag = 0;
12599 {
12600 local_req_flag = 1;
12601 }
12603 &req_flag,
12604 1,
12606 MPI_MAX,
12607 Communicator_pt->mpi_comm());
12609 if (req_flag == 1)
12610 {
12612 }
12613
12614 // If refine and prune is required make number of uniform
12615 // refinements for each mesh consistent otherwise code
12616 // hangs on "empty" processors for which no restart file exists
12618 {
12619 // This is what we have locally
12621 // Synchronise over all processors with max operation
12624 n_mesh,
12626 MPI_MAX,
12627 Communicator_pt->mpi_comm());
12628
12629#ifdef PARANOID
12630 // Check it: Reconciliation should only be required for
12631 // for processors on which no restart file was opened and
12632 // for which the meshes are therefore empty
12633 bool fail = false;
12634 std::ostringstream error_message;
12635 error_message << "Number of uniform refinements was not consistent \n"
12636 << "for following meshes during restart on processor \n"
12637 << "on which restart file could be opened:\n";
12638 for (unsigned i = 0; i < n_mesh; i++)
12639 {
12642 {
12643 fail = true;
12644 error_message << "Sub-mesh: " << i << "; local nrefinement: "
12645 << local_nrefinement_for_mesh[i] << " "
12646 << "; global/synced nrefinement: "
12647 << nrefinement_for_mesh[i] << "\n";
12648 }
12649 }
12650 if (fail)
12651 {
12653 error_message.str(), "Problem::read()", OOMPH_EXCEPTION_LOCATION);
12654 }
12655#endif
12656 }
12657 }
12658#endif
12659
12660 // Read line up to termination sign
12662
12663 // Ignore rest of line
12664 restart_file.ignore(80, '\n');
12665
12666 // Check flag that indicates that we've read the final data
12667 unsigned tmp;
12668 tmp = std::atoi(input_string.c_str());
12669
12670#ifdef PARANOID
12672 {
12673 if (tmp != 9999)
12674 {
12675 std::ostringstream error_message;
12676 error_message
12677 << "Error in reading restart data: Uniform refinement when pruned \n"
12678 << "flags should be followed by 9999.\n";
12679 throw OomphLibError(error_message.str(),
12682 }
12683 }
12684
12685#else
12686 // Suppress comiler warnings about non-used variable
12687 tmp++;
12688 tmp--;
12689#endif
12690
12691
12692#ifdef OOMPH_HAS_MPI
12693
12694 // Refine and prune if required
12696 {
12699 }
12700
12701 // target_domain_for_local_non_halo_element[e] contains the number
12702 // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
12703 // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
12704 // elements is the same as in the Problem's mesh, with the halo
12705 // elements being skipped.
12707
12708 // If a restart file has been generated using code compiled without MPI
12709 // then it will not have any of the base element data.
12710 // If we try to read in that file with code that has been compied using
12711 // MPI, even if running only one processor, then it will fail here.
12712 // The ideal fix is to edit the restart file so that it contains the two
12713 // lines
12714 //
12715 // 0 # Number of base elements; partitioning follows.
12716 // 8888 # Test flag for end of base element distribution
12717 //
12718 // after the end of the sub-meshes, but before the number of elements
12719 // However, we can determine that this is the problem if n_base = 0,
12720 // so there is a little bit of logic below to catch this case
12721
12722 // Store current location in the file (before we are about to read
12723 // in either the base mesh or number of elements of the first mesh)
12724 std::streampos position_before_base_element = restart_file.tellg();
12725 // Boolean flag used to set whether to read in base element info
12726 bool read_in_base_element_info = true;
12727
12728 // Read line up to termination sign
12730
12731 // Ignore rest of line
12732 restart_file.ignore(80, '\n');
12733
12734 // Get number of base elements as recorded
12735 unsigned n_base_element_read_in = atoi(input_string.c_str());
12736 unsigned nbase = Base_mesh_element_pt.size();
12738 {
12740 {
12741 // If we have zero base elements the problem could be that the
12742 // restart file was generated without MPI. Issue a warning
12743 // and continue anyway
12744 if (nbase == 0)
12745 {
12746 std::ostringstream warn_message;
12748 << "The number of base elements in the mesh is 0,\n"
12749 << " but the restart file indicates that there are "
12750 << n_base_element_read_in << ".\n"
12751 << "This could be because the restart file was \n"
12752 << "generated by using code without MPI.\n"
12753 << "\n"
12754 << "The best fix is to include two additional lines\n"
12755 << "in the restart file: \n\n"
12756 << "0 # Number of base elements; partitioning follows.\n"
12757 << "8888 # Test flag for end of base element distribution\n"
12758 << "\n"
12759 << "These lines go after the flag 9999 that indicates\n"
12760 << "the end of the submesh information.\n"
12761 << "\n"
12762 << "The file will now continue to be read assuming that\n"
12763 << "the base element information is not present.\n"
12764 << "If you get strange results then please look carefully\n"
12765 << "at the restart file. The safest thing to do is to \n"
12766 << "ensure that the restart file was generated by code\n"
12767 << "compiled and run with the same parallel options.\n";
12771 // Set the skip flag to true
12772 // and rewind the file pointer
12775 }
12776 // Otherwise throw a hard error
12777 else
12778 {
12779 std::ostringstream error_message;
12780 error_message << "About to read " << n_base_element_read_in
12781 << " base elements \n"
12782 << "though we only have " << nbase
12783 << " base elements in mesh.\n";
12784 throw OomphLibError(error_message.str(),
12787 }
12788 }
12789 }
12790
12791 // Read in the remaning base element information, if necessary
12792 if (read_in_base_element_info == true)
12793 {
12794 // Read in target_domain_for_base_element[e] for all base elements
12796 for (unsigned e = 0; e < nbase; e++)
12797 {
12798 // Read line
12800
12801 // Get target domain
12803 }
12804
12805 // Read line up to termination sign
12807
12808 // Ignore rest of line
12809 restart_file.ignore(80, '\n');
12810
12811 // Check flag that indicates that we've read the final data
12812 tmp = std::atoi(input_string.c_str());
12813
12814
12815#ifdef PARANOID
12817 {
12818 if (tmp != 8888)
12819 {
12820 std::ostringstream error_message;
12821 error_message
12822 << "Error in reading restart data: Target proc for base elements \n"
12823 << "should be followed by 8888.\n";
12824 throw OomphLibError(error_message.str(),
12827 }
12828 }
12829#endif
12830
12831 // Loop over all elements (incl. any FaceElements) and assign
12832 // target domain for all local non-halo elements and check if
12833 // load balancing is required -- no need to do this if problem is
12834 // not distributed.
12835 unsigned load_balance_required_flag = 0;
12837 {
12838 // Working with TreeBasedRefineableMeshBase mesh
12840 if (dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12841 {
12842 const int my_rank = this->communicator_pt()->my_rank();
12843 unsigned nel = mesh_pt()->nelement();
12844 for (unsigned e = 0; e < nel; e++)
12845 {
12847 if (!el_pt->is_halo())
12848 {
12849 // Get element number (plus one) in base element enumeration
12852
12853 // If it's zero then we haven't found it, it may be a FaceElement
12854 // (in which case we move it to the same processor as its bulk
12855 // element
12857 {
12858 FaceElement* face_el_pt = dynamic_cast<FaceElement*>(el_pt);
12859 if (face_el_pt != 0)
12860 {
12861 // Get corresponding bulk element
12862 FiniteElement* bulk_el_pt = face_el_pt->bulk_element_pt();
12863
12864 // Use its element number (plus one) in base element
12865 // enumeration
12868
12869 // If this is zero too we have a problem
12871 {
12872 throw OomphLibError(
12873 "el_number_in_base_mesh_plus_one=0 for bulk",
12874 "Problem::read()",
12876 }
12877 }
12878 }
12879
12880 // If we've made it here then we're not dealing with a
12881 // FaceElement but with an element that doesn't exist locally
12882 // --> WTF?
12884 {
12885 throw OomphLibError("el_number_in_base_mesh_plus_one=0",
12888 }
12889
12890 // Assign target domain for next local non-halo element in
12891 // the order in which it's encountered in the global mesh
12894 1]);
12895
12896 // Do elements on this processor to be moved elsewhere?
12899 {
12901 }
12902 }
12903 }
12904
12905 } // if (working with TreeBasedRefineableMeshBase mesh)
12906
12907 // Get overall need to load balance by max
12910 1,
12912 MPI_MAX,
12913 this->communicator_pt()->mpi_comm());
12914 }
12915
12916 // Do we need to load balance?
12918 {
12919 oomph_info << "Doing load balancing after pruning\n";
12920 DocInfo doc_info;
12921 doc_info.disable_doc();
12922 bool report_stats = false;
12925 oomph_info << "Done load balancing after pruning\n";
12926 }
12927 else
12928 {
12929 oomph_info << "No need for load balancing after pruning\n";
12930 }
12931 } // End of read in base element information
12932#endif
12933
12934
12935 // Boolean to record if any unstructured bulk meshes have
12936 // been read in (and therefore completely re-generated, with new
12937 // elements and nodes) from disk
12938 bool have_read_unstructured_mesh = false;
12939
12940 // Call the actions before adaptation
12942
12943 // If there are unstructured meshes in the problem we need
12944 // to strip out any face elements that are attached to them
12945 // because restart of unstructured meshes re-creates their elements
12946 // and nodes from scratch, leading to dangling pointers from the
12947 // face elements to the old elements and nodes. This function is
12948 // virtual and (practically) empty in the Problem base class
12949 // but toggles a flag to indicate that it has been called. We can then
12950 // issue a warning below, prompting the user to consider overloading it
12951 // if the problem is found to contain unstructured bulk meshes.
12952 // Warning can be ignored if the bulk mesh is not associated with any
12953 // face elements.
12956
12957 // Update number of submeshes
12958 n_mesh = nsub_mesh();
12959
12960 // Single mesh:
12961 //------------
12962 if (n_mesh == 0)
12963 {
12964 // Refine single mesh (if it's refineable)
12966 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
12967 {
12968 // When we get in here the problem has been constructed
12969 // by the constructor and the mesh is its original unrefined
12970 // form.
12971 // RefineableMeshBase::refine(...) reads the refinement pattern from the
12972 // specified file and performs refinements until the mesh has
12973 // reached the same level of refinement as the mesh that existed
12974 // when the problem was dumped to disk.
12975 mmesh_pt->refine(restart_file);
12976 }
12977#ifdef OOMPH_HAS_TRIANGLE_LIB
12978 // Regenerate mesh from triangulate IO if it's a triangular mesh
12979 TriangleMeshBase* mmesh_pt = dynamic_cast<TriangleMeshBase*>(mesh_pt(0));
12980 if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
12981 {
12982#ifdef OOMPH_HAS_MPI
12983 // Check if the mesh is distributed, if that is the case then
12984 // additional info. needs to be read
12985 if (mmesh_pt->is_mesh_distributed())
12986 {
12987 // Dump the info. related with the distribution of the mesh
12988 mmesh_pt->read_distributed_info_for_restart(restart_file);
12989 }
12990#endif
12991 // The function reads the TriangulateIO data structure from the dump
12992 // file and then completely regenerates the mesh using the
12993 // data structure
12994 mmesh_pt->remesh_from_triangulateio(restart_file);
12996#ifdef OOMPH_HAS_MPI
12997 // Check if the mesh is distributed, if that is the case then we
12998 // need to re-establish the halo/haloed scheme (similar as in the
12999 // RefineableTriangleMesh::adapt() method)
13000 if (mmesh_pt->is_mesh_distributed())
13001 {
13002 mmesh_pt->reestablish_distribution_info_for_restart(
13003 this->communicator_pt(), restart_file);
13004 }
13005#endif
13006 // Still left to update the polylines representation, that is performed
13007 // later since the nodes positions may still change when reading info.
13008 // for the mesh, see below
13009 }
13010#endif
13011 }
13012
13013 // Multiple submeshes
13014 //------------------
13015 else
13016 {
13017 // Loop over submeshes
13018 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
13019 {
13020 // Refine single mesh (if its refineable)
13022 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(imesh)))
13023 {
13024 // When we get in here the problem has been constructed
13025 // by the constructor and the mesh is its original unrefined
13026 // form.
13027 // RefineableMeshBase::refine(...) reads the refinement pattern from
13028 // the specified file and performs refinements until the mesh has
13029 // reached the same level of refinement as the mesh that existed
13030 // when the problem was dumped to disk.
13031 mmesh_pt->refine(restart_file);
13032 }
13033#ifdef OOMPH_HAS_TRIANGLE_LIB
13034 // Regenerate mesh from triangulate IO if it's a triangular mesh
13036 dynamic_cast<TriangleMeshBase*>(mesh_pt(imesh));
13037 if (mmesh_pt != 0 && mmesh_pt->use_triangulateio_restart())
13038 {
13039#ifdef OOMPH_HAS_MPI
13040 // Check if the mesh is distributed, if that is the case then
13041 // additional info. needs to be read
13042 if (mmesh_pt->is_mesh_distributed())
13043 {
13044 // Dump the info. related with the distribution of the mesh
13045 mmesh_pt->read_distributed_info_for_restart(restart_file);
13046 }
13047#endif
13048 // The function reads the TriangulateIO data structure from the dump
13049 // file and then completely regenerates the mesh using the
13050 // data structure
13051 mmesh_pt->remesh_from_triangulateio(restart_file);
13053
13054#ifdef OOMPH_HAS_MPI
13055 // Check if the mesh is distributed, if that is the case then we
13056 // need to re-establish the halo/haloed scheme (similar as in the
13057 // RefineableTriangleMesh::adapt() method)
13058 if (mmesh_pt->is_mesh_distributed())
13059 {
13060 mmesh_pt->reestablish_distribution_info_for_restart(
13061 this->communicator_pt(), restart_file);
13062 }
13063#endif
13064 // Still left to update the polylines representation, that is
13065 // performed later since the nodes positions may still change when
13066 // reading info. for the mesh, see below
13067 }
13068#endif
13069 } // End of loop over submeshes
13070
13071
13072 // Rebuild the global mesh
13074 }
13075
13076 // Any actions after adapt
13078
13079 // Re-attach face elements (or whatever else needs to be done
13080 // following the total re-generation of the unstructured meshes
13083
13084
13085 // Issue warning:
13087 {
13089 {
13092 {
13093 std::ostringstream warn_message;
13095 << "I've just read in some unstructured meshes and have, in\n"
13096 << "the process, totally re-generated their nodes and elements.\n"
13097 << "This may create dangling pointers that still point to the\n"
13098 << "old nodes and elements, e.g. because FaceElements were\n"
13099 << "attached to these meshes or pointers to nodes and elements\n"
13100 << "were stored somewhere. FaceElements should therefore be\n"
13101 << "removed before reading in these meshes, using an overloaded\n"
13102 << "version of the function\n\n"
13103 << " Problem::actions_before_read_unstructured_meshes()\n\n"
13104 << "and then re-attached using an overloaded version of\n\n"
13105 << " Problem::actions_after_read_unstructured_meshes().\n\n"
13106 << "The required content of these functions is likely to be "
13107 "similar\n"
13108 << "to the Problem::actions_before_adapt() and \n"
13109 << "Problem::actions_after_adapt() that would be required in\n"
13110 << "a spatially adaptive computation. If these functions already\n"
13111 << "exist and perform the required actions, the \n"
13112 << "actions_before/after_read_unstructured_meshes() functions\n"
13113 << "can remain empty because the former are called automatically.\n"
13114 << "In this case, this warning my be suppressed by setting the\n"
13115 << "public boolean\n\n"
13116 << " "
13117 "Problem::Suppress_warning_about_actions_before_read_"
13118 "unstructured_meshes\n\n"
13119 << "to true." << std::endl;
13123 }
13124 }
13125 }
13126
13127 // Setup equation numbering scheme
13128 oomph_info << "\nNumber of equations in Problem::read(): "
13129 << assign_eqn_numbers() << std::endl
13130 << std::endl;
13131 // Read time info
13132 //---------------
13133 unsigned local_unsteady_restart_flag = 0;
13134 double local_time = -DBL_MAX;
13135 unsigned local_n_dt = 0;
13136#ifdef OOMPH_HAS_MPI
13137 unsigned local_sync_needed_flag = 0;
13138#endif
13140
13141 if (restart_file.is_open())
13142 {
13143 oomph_info << "Restart file exists" << std::endl;
13144#ifdef OOMPH_HAS_MPI
13146#endif
13147 // Read line up to termination sign
13149
13150 // Ignore rest of line
13151 restart_file.ignore(80, '\n');
13152
13153 // Is the restart data from an unsteady run?
13155
13156 // Read line up to termination sign
13158
13159 // Ignore rest of line
13160 restart_file.ignore(80, '\n');
13161
13162 // Read in initial time and set
13163 local_time = atof(input_string.c_str());
13164
13165 // Read line up to termination sign
13167
13168 // Ignore rest of line
13169 restart_file.ignore(80, '\n');
13170
13171 // Read & set number of timesteps
13172 local_n_dt = atoi(input_string.c_str());
13173 local_dt.resize(local_n_dt);
13174
13175 // Read in timesteps:
13176 for (unsigned i = 0; i < local_n_dt; i++)
13177 {
13178 // Read line up to termination sign
13180
13181 // Ignore rest of line
13182 restart_file.ignore(80, '\n');
13183
13184 // Read in initial time and set
13185 double prev_dt = atof(input_string.c_str());
13186 local_dt[i] = prev_dt;
13187 }
13188 }
13189 else
13190 {
13191 oomph_info << "Restart file does not exist" << std::endl;
13192#ifdef OOMPH_HAS_MPI
13194#endif
13195 }
13196
13197
13198 // No prepare global values, possibly via sync
13199 Vector<double> dt;
13200
13201 // Do we need to sync?
13202 unsigned sync_needed_flag = 0;
13203
13204#ifdef OOMPH_HAS_MPI
13206 {
13207 // Get need to sync by max
13210 1,
13212 MPI_MAX,
13213 this->communicator_pt()->mpi_comm());
13214 }
13215#endif
13216
13217 // Synchronise
13218 if (sync_needed_flag == 1)
13219 {
13220#ifdef OOMPH_HAS_MPI
13221
13222
13223#ifdef PARANOID
13225 {
13226 std::ostringstream error_message;
13227 error_message << "Synchronisation of temporal restart data \n"
13228 << "required even though Problem hasn't been distributed "
13229 "-- very odd!\n";
13230 throw OomphLibError(error_message.str(),
13233 }
13234#endif
13235
13236 // Get unsteady restart flag by max-based reduction
13237 unsigned unsteady_restart_flag = 0;
13240 1,
13242 MPI_MAX,
13243 this->communicator_pt()->mpi_comm());
13244
13245 // So, is it an unsteady restart?
13246 unsteady_restart = false;
13247 if (unsteady_restart_flag == 1)
13248 {
13249 unsteady_restart = true;
13250
13251 // Get time by max
13252 double time = -DBL_MAX;
13254 &time,
13255 1,
13256 MPI_DOUBLE,
13257 MPI_MAX,
13258 this->communicator_pt()->mpi_comm());
13259 time_pt()->time() = time;
13260
13261 // Get number of timesteps by max-based reduction
13262 unsigned n_dt = 0;
13264 &n_dt,
13265 1,
13267 MPI_MAX,
13268 this->communicator_pt()->mpi_comm());
13269
13270 // Resize whatever needs resizing
13271 time_pt()->resize(n_dt);
13272 dt.resize(n_dt);
13273 if (local_dt.size() == 0)
13274 {
13275 local_dt.resize(n_dt, -DBL_MAX);
13276 }
13277
13278 // Get timesteps increments by max-based reduction
13280 &dt[0],
13281 n_dt,
13282 MPI_DOUBLE,
13283 MPI_MAX,
13284 this->communicator_pt()->mpi_comm());
13285 }
13286
13287#else
13288
13289 std::ostringstream error_message;
13290 error_message
13291 << "Synchronisation of temporal restart data \n"
13292 << "required even though we don't have mpi support -- very odd!\n";
13293 throw OomphLibError(
13294 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
13295
13296#endif
13297 }
13298 // No sync needed -- just copy across
13299 else
13300 {
13301 unsteady_restart = false;
13303 {
13304 unsteady_restart = true;
13305 time_pt()->time() = local_time;
13307 dt.resize(local_n_dt);
13308 for (unsigned i = 0; i < local_n_dt; i++)
13309 {
13310 dt[i] = local_dt[i];
13311 }
13312 }
13313 }
13314
13315 // Initialise timestep -- also sets the weights for all timesteppers
13316 // in the problem.
13318
13319 // Loop over submeshes:
13320 unsigned nmesh = nsub_mesh();
13321 if (nmesh == 0) nmesh = 1;
13322 for (unsigned m = 0; m < nmesh; m++)
13323 {
13324 // //---------------------------------------------------------
13325 // // Keep this commented out code around to debug restarts
13326 // //---------------------------------------------------------
13327 // std::ofstream some_file;
13328 // char filename[100];
13329 // sprintf(filename,"read_mesh%i_on_proc%i.dat",m,
13330 // this->communicator_pt()->my_rank());
13331 // some_file.open(filename);
13332 // mesh_pt(m)->output(some_file);
13333 // some_file.close();
13334
13335 // sprintf(filename,"read_mesh%i_with_haloes_on_proc%i.dat",m,
13336 // this->communicator_pt()->my_rank());
13337 // mesh_pt(m)->enable_output_of_halo_elements();
13338 // some_file.open(filename);
13339 // mesh_pt(m)->output(some_file);
13340 // mesh_pt(m)->disable_output_of_halo_elements();
13341 // some_file.close();
13342 // oomph_info << "Doced mesh " << m << " before reading\n";
13343
13344 // sprintf(filename,"read_nodes_mesh%i_on_proc%i.dat",m,
13345 // this->communicator_pt()->my_rank());
13346 // some_file.open(filename);
13347 // unsigned nnod=mesh_pt(m)->nnode();
13348 // for (unsigned j=0;j<nnod;j++)
13349 // {
13350 // Node* nod_pt=mesh_pt(m)->node_pt(j);
13351 // unsigned n=nod_pt->ndim();
13352 // for (unsigned i=0;i<n;i++)
13353 // {
13354 // some_file << nod_pt->x(i) << " ";
13355 // }
13356 // some_file << nod_pt->is_halo() << " "
13357 // << nod_pt->nvalue() << " "
13358 // << nod_pt->hang_code() << "\n";
13359 // }
13360 // some_file.close();
13361 // oomph_info << "Doced mesh " << m << " before reading\n";
13362 // //---------------------------------------------------------
13363 // // End keep this commented out code around to debug restarts
13364 // //---------------------------------------------------------
13365
13367
13368#ifdef OOMPH_HAS_TRIANGLE_LIB
13369 // Here update the polyline representation if working with
13370 // triangle base meshes
13372 dynamic_cast<TriangleMeshBase*>(mesh_pt(m)))
13373 {
13374 // In charge of updating the polylines representation to the
13375 // current refinement/unrefinement level after restart, it
13376 // also update the shared boundaries in case of working with a
13377 // distributed mesh
13378 mmesh_pt->update_polyline_representation_from_restart();
13379 }
13380#endif // #ifdef OOMPH_HAS_TRIANGLE_LIB
13381 }
13382
13383 // Read global data:
13384 //------------------
13385
13386 // Number of global data
13387 unsigned Nglobal = Global_data_pt.size();
13388
13389 // Read line up to termination sign
13391
13392 // Ignore rest of line
13393 restart_file.ignore(80, '\n');
13394
13395 // Check # of nodes:
13396 unsigned long check_nglobal = atoi(input_string.c_str());
13397
13398
13400 {
13401 if (check_nglobal != Nglobal)
13402 {
13403 std::ostringstream error_message;
13404 error_message << "The number of global data " << Nglobal
13405 << " is not equal to that specified in the input file "
13406 << check_nglobal << std::endl;
13407
13408 throw OomphLibError(error_message.str(),
13411 }
13412 }
13413
13414 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13415 {
13417 }
13418 }
13419
13420 //===================================================================
13421 /// Set all timesteps to the same value, dt, and assign
13422 /// weights for all timesteppers in the problem.
13423 //===================================================================
13424 void Problem::initialise_dt(const double& dt)
13425 {
13426 // Initialise the timesteps in the Problem's time object
13428
13429 // Find out how many timesteppers there are
13430 unsigned n_time_steppers = ntime_stepper();
13431
13432 // Loop over them all and set the weights
13433 for (unsigned i = 0; i < n_time_steppers; i++)
13434 {
13436 if (time_stepper_pt(i)->adaptive_flag())
13437 {
13439 }
13440 }
13441 }
13442
13443 //=========================================================================
13444 /// Set the value of the timesteps to be equal to the values passed in
13445 /// a vector and assign weights for all timesteppers in the problem
13446 //========================================================================
13448 {
13449 // Initialise the timesteps in the Problem's time object
13451
13452 // Find out how many timesteppers there are
13453 unsigned n_time_steppers = ntime_stepper();
13454
13455 // Loop over them all and set the weights
13456 for (unsigned i = 0; i < n_time_steppers; i++)
13457 {
13459 if (time_stepper_pt(i)->adaptive_flag())
13460 {
13462 }
13463 }
13464 }
13465
13466 //========================================================
13467 /// Self-test: Check meshes and global data. Return 0 for OK
13468 //========================================================
13470 {
13471 // Initialise
13472 bool passed = true;
13473
13474 // Are there any submeshes?
13475 unsigned Nmesh = nsub_mesh();
13476
13477 // Just one mesh: Check it
13478 if (Nmesh == 0)
13479 {
13480 if (mesh_pt()->self_test() != 0)
13481 {
13482 passed = false;
13484 << "\n ERROR: Failed Mesh::self_test() for single mesh in problem"
13485 << std::endl;
13486 }
13487 }
13488 // Loop over all submeshes and check them
13489 else
13490 {
13491 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
13492 {
13493 if (mesh_pt(imesh)->self_test() != 0)
13494 {
13495 passed = false;
13496 oomph_info << "\n ERROR: Failed Mesh::self_test() for mesh imesh"
13497 << imesh << std::endl;
13498 }
13499 }
13500 }
13501
13502
13503 // Check global data
13504 unsigned Nglobal = Global_data_pt.size();
13505 for (unsigned iglobal = 0; iglobal < Nglobal; iglobal++)
13506 {
13507 if (Global_data_pt[iglobal]->self_test() != 0)
13508 {
13509 passed = false;
13511 << "\n ERROR: Failed Data::self_test() for global data iglobal"
13512 << iglobal << std::endl;
13513 }
13514 }
13515
13516
13517#ifdef OOMPH_HAS_MPI
13518
13520 {
13521 // Note: This throws an error if it fails so no return is required.
13523 tmp_doc_info.disable_doc();
13525 }
13526
13527#endif
13528
13529 // Return verdict
13530 if (passed)
13531 {
13532 return 0;
13533 }
13534 else
13535 {
13536 return 1;
13537 }
13538 }
13539
13540 //====================================================================
13541 /// A function that is used to adapt a bifurcation-tracking
13542 /// problem, which requires separate interpolation of the
13543 /// associated eigenfunction. The error measure is chosen to be
13544 /// a suitable combination of the errors in the base flow and the
13545 /// eigenfunction. The bifurcation type is passed as an argument
13546 //=====================================================================
13548 unsigned& n_unrefined,
13549 const unsigned& bifurcation_type,
13550 const bool& actually_adapt)
13551 {
13552 // Storage for eigenfunction from the problem
13554 // Get the eigenfunction from the problem
13555 this->get_bifurcation_eigenfunction(eigenfunction);
13556
13557 // Get the bifurcation parameter
13558 double* parameter_pt = this->bifurcation_parameter_pt();
13559
13560 // Get the frequency parameter if tracking a Hopf bifurcation
13561 double omega = 0.0;
13562 // If we're tracking a Hopf then also get the frequency
13563 if (bifurcation_type == 3)
13564 {
13565 omega = dynamic_cast<HopfHandler*>(assembly_handler_pt())->omega();
13566 }
13567
13568 // If we're tracking a Pitchfork get the slack parameter (Hack)
13569 double sigma = 0.0;
13570 if (bifurcation_type == 2)
13571 {
13572 sigma = this->dof(this->ndof() - 1);
13573 }
13574
13575 // We can now deactivate the bifurcation tracking in the problem
13576 // to restore the degrees of freedom to the unaugmented value
13578
13579 // Next, we create copies of the present problem
13580 // The number of copies depends on the number of eigenfunctions
13581 // One copy for each eigenfunction
13582 const unsigned n_copies = eigenfunction.size();
13584
13585 // Loop over the number of copies
13586 for (unsigned c = 0; c < n_copies; c++)
13587 {
13588 // If we don't already have a copy
13589 if (Copy_of_problem_pt[c] == 0)
13590 {
13591 // Create the copy
13592 Copy_of_problem_pt[c] = this->make_copy();
13593
13594 // Refine the copy to the same level as the current problem
13595
13596 // Find number of submeshes
13597 const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13598 // If there is only one mesh
13599 if (N_mesh == 0)
13600 {
13601 // Can we refine the mesh
13603 dynamic_cast<TreeBasedRefineableMeshBase*>(
13605 {
13606 // Is the adapt flag set
13607 if (mmesh_pt->is_adaptation_enabled())
13608 {
13609 // Now get the original problem's mesh if it's refineable
13611 dynamic_cast<TreeBasedRefineableMeshBase*>(
13612 this->mesh_pt(0)))
13613 {
13614 mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13616 }
13617 else
13618 {
13620 << "Info/Warning: Mesh in orginal problem is not refineable."
13621 << std::endl;
13622 }
13623 }
13624 else
13625 {
13626 oomph_info << "Info/Warning: Mesh adaptation is disabled in copy."
13627 << std::endl;
13628 }
13629 }
13630 else
13631 {
13632 oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13633 << std::endl;
13634 }
13635 } // End of single mesh case
13636 // Otherwise loop over the submeshes
13637 else
13638 {
13639 for (unsigned m = 0; m < N_mesh; m++)
13640 {
13641 // Can we refine the submesh
13643 dynamic_cast<TreeBasedRefineableMeshBase*>(
13645 {
13646 // Is the adapt flag set
13647 if (mmesh_pt->is_adaptation_enabled())
13648 {
13649 // Now get the original problem's mesh
13651 dynamic_cast<TreeBasedRefineableMeshBase*>(
13652 this->mesh_pt(m)))
13653 {
13654 mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13656 }
13657 else
13658 {
13659 oomph_info << "Info/Warning: Mesh in orginal problem is not "
13660 "refineable."
13661 << std::endl;
13662 }
13663 }
13664 else
13665 {
13667 << "Info/Warning: Mesh adaptation is disabled in copy."
13668 << std::endl;
13669 }
13670 }
13671 else
13672 {
13673 oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13674 << std::endl;
13675 }
13676 }
13677 // rebuild the global mesh in the copy
13678 Copy_of_problem_pt[c]->rebuild_global_mesh();
13679
13680 } // End of multiple mesh case
13681
13682 // Must call actions after adapt
13683 Copy_of_problem_pt[c]->actions_after_adapt();
13684
13685 // Assign the equation numbers to the copy (quietly)
13687 }
13688 } // End of creation of copies
13689
13690
13691 // Now check some numbers
13692 for (unsigned c = 0; c < n_copies; c++)
13693 {
13694 // Check that the dofs match for each copy
13695#ifdef PARANOID
13696 // If the problems don't match then complain
13697 if (Copy_of_problem_pt[c]->ndof() != this->ndof())
13698 {
13699 std::ostringstream error_stream;
13700 error_stream << "Number of unknowns in the problem copy " << c << " "
13701 << "not equal to number in the original:\n"
13702 << this->ndof() << " (original) "
13703 << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
13704
13705 throw OomphLibError(
13707 }
13708#endif
13709
13710 // Assign the eigenfunction(s) to the copied problems
13711 Copy_of_problem_pt[c]->assign_eigenvector_to_dofs(eigenfunction[c]);
13712 // Set all pinned values to zero
13713 Copy_of_problem_pt[c]->set_pinned_values_to_zero();
13714 }
13715
13716 // Symmetrise the problem if we are solving a pitchfork
13717 if (bifurcation_type == 2)
13718 {
13720 ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13721 }
13722
13723 // Find error estimates based on current problem and eigenproblem
13724 // Now we need to get the error estimates for both problems.
13726 this->get_all_error_estimates(base_error);
13727 // Loop over the copies
13728 for (unsigned c = 0; c < n_copies; c++)
13729 {
13730 // Get the error estimates for the copy
13731 Copy_of_problem_pt[c]->get_all_error_estimates(eigenfunction_error);
13732
13733 // Find the number of meshes
13734 unsigned n_mesh = base_error.size();
13735
13736#ifdef PARANOID
13738 {
13739 std::ostringstream error_stream;
13740 error_stream << "Problems do not have the same number of meshes\n"
13741 << "Base : " << n_mesh
13742 << " : Eigenproblem : " << eigenfunction_error.size()
13743 << "\n";
13744 throw OomphLibError(
13746 }
13747#endif
13748
13749 for (unsigned m = 0; m < n_mesh; m++)
13750 {
13751 // Check the number of elements is the same
13752 unsigned n_element = base_error[m].size();
13753#ifdef PARANOID
13754 if (n_element != eigenfunction_error[m].size())
13755 {
13756 std::ostringstream error_stream;
13757 error_stream << "Mesh " << m
13758 << " does not have the same number of elements in the "
13759 "two problems:\n"
13760 << "Base: " << n_element
13761 << " : Eigenproblem: " << eigenfunction_error[m].size()
13762 << "\n";
13763 throw OomphLibError(error_stream.str(),
13766 }
13767#endif
13768 // Now add all the error esimates together
13769 for (unsigned e = 0; e < n_element; e++)
13770 {
13771 // Add the error estimates (lazy)
13773 }
13774 }
13775 } // End of loop over copies
13776
13777 // Then refine all problems based on the combined measure
13778 // if we are actually adapting (not just estimating the errors)
13779 if (actually_adapt)
13780 {
13782 for (unsigned c = 0; c < n_copies; c++)
13783 {
13784 Copy_of_problem_pt[c]->adapt_based_on_error_estimates(
13786 }
13787 // Symmetrise the problem (again) if we are solving for a pitchfork
13788 if (bifurcation_type == 2)
13789 {
13791 ->symmetrise_eigenfunction_for_adaptive_pitchfork_tracking();
13792 }
13793
13794 // Now get the refined guess for the eigenvector
13795 for (unsigned c = 0; c < n_copies; c++)
13796 {
13797 Copy_of_problem_pt[c]->get_dofs(eigenfunction[c]);
13798 }
13799 }
13800
13801 // Reactivate the tracking
13802 switch (bifurcation_type)
13803 {
13804 // Fold tracking
13805 case 1:
13806 this->activate_fold_tracking(parameter_pt);
13807 break;
13808
13809 // Pitchfork
13810 case 2:
13811 this->activate_pitchfork_tracking(parameter_pt, eigenfunction[0]);
13812 // reset the slack parameter
13813 this->dof(this->ndof() - 1) = sigma;
13814 break;
13815
13816 // Hopf
13817 case 3:
13819 parameter_pt, omega, eigenfunction[0], eigenfunction[1]);
13820 break;
13821
13822 default:
13823 std::ostringstream error_stream;
13824 error_stream << "Bifurcation type " << bifurcation_type
13825 << " not known\n"
13826 << "1: Fold, 2: Pitchfork, 3: Hopf\n";
13827 throw OomphLibError(
13829 }
13830 }
13831
13832
13833 //====================================================================
13834 /// A function that is used to document the errors when
13835 /// adapting a bifurcation-tracking
13836 /// problem, which requires separate interpolation of the
13837 /// associated eigenfunction. The error measure is chosen to be
13838 /// a suitable combination of the errors in the base flow and the
13839 /// eigenfunction. The bifurcation type is passed as an argument
13840 //=====================================================================
13841 void Problem::bifurcation_adapt_doc_errors(const unsigned& bifurcation_type)
13842 {
13843 // Dummy arguments
13844 unsigned n_refined, n_unrefined;
13845 // Just call the bifurcation helper without actually adapting
13846 bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type, false);
13847 }
13848
13849
13850 //========================================================================
13851 /// Adapt problem:
13852 /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
13853 /// based on their own error estimates and the target errors specified
13854 /// in the mesh(es). Following mesh adaptation,
13855 /// update global mesh, and re-assign equation numbers.
13856 /// Return # of refined/unrefined elements. On return from this
13857 /// function, Problem can immediately be solved again.
13858 //======================================================================
13859 void Problem::adapt(unsigned& n_refined, unsigned& n_unrefined)
13860 {
13861 double t_start_total = 0.0;
13863 {
13865 }
13866
13867 // Get the bifurcation type
13868 int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
13869
13870 bool continuation_problem = false;
13871
13872 // If we have continuation data then we need to project that across to the
13873 // new mesh
13875 {
13876 if (Dof_derivative.size() != 0)
13877 {
13878 continuation_problem = true;
13879 }
13880 }
13881
13882 // If we are tracking a bifurcation then call the bifurcation adapt function
13883 if (bifurcation_type != 0)
13884 {
13885 this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
13886 // Return immediately
13887 return;
13888 }
13889
13891 {
13892 // Create a copy of the problem
13893 Copy_of_problem_pt.resize(2);
13894 // If we don't already have a copy
13895 for (unsigned c = 0; c < 2; c++)
13896 {
13897 if (Copy_of_problem_pt[c] == 0)
13898 {
13899 // Create the copy
13900 Copy_of_problem_pt[c] = this->make_copy();
13901
13902 // Refine the copy to the same level as the current problem
13903 // Must call actions before adapt
13904 Copy_of_problem_pt[c]->actions_before_adapt();
13905
13906 // Find number of submeshes
13907 const unsigned N_mesh = Copy_of_problem_pt[c]->nsub_mesh();
13908
13909 // If there is only one mesh
13910 if (N_mesh == 0)
13911 {
13912 // Can we refine the mesh
13914 dynamic_cast<TreeBasedRefineableMeshBase*>(
13916 {
13917 // Is the adapt flag set
13918 if (mmesh_pt->is_adaptation_enabled())
13919 {
13920 // Now get the original problem's mesh if it's refineable
13922 dynamic_cast<TreeBasedRefineableMeshBase*>(
13923 this->mesh_pt(0)))
13924 {
13925 if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13926 {
13928 << "Info/Warning: Adaptive Continuation is broken in "
13929 << "SolidElement" << std::endl;
13930 }
13931 mmesh_pt->refine_base_mesh_as_in_reference_mesh(
13933 }
13934 else
13935 {
13936 oomph_info << "Info/Warning: Mesh in orginal problem is not "
13937 "refineable."
13938 << std::endl;
13939 }
13940 }
13941 else
13942 {
13944 << "Info/Warning: Mesh adaptation is disabled in copy."
13945 << std::endl;
13946 }
13947 }
13948 else if (TriangleMeshBase* tmesh_pt =
13949 dynamic_cast<TriangleMeshBase*>(
13951 {
13953 dynamic_cast<TriangleMeshBase*>(this->mesh_pt(0)))
13954 {
13955 if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
13956 {
13958 << "Info/Warning: Adaptive Continuation is broken in "
13959 << "SolidElement" << std::endl;
13960 }
13961
13962 // Remesh using the triangulateIO of the base mesh
13963 // Done via a file, so a bit hacky but this will be
13964 // superseded very soon
13965 std::ofstream tri_dump("triangle_mesh.dmp");
13966 original_mesh_pt->dump_triangulateio(tri_dump);
13967 tri_dump.close();
13968 std::ifstream tri_read("triangle_mesh.dmp");
13969 tmesh_pt->remesh_from_triangulateio(tri_read);
13970 tri_read.close();
13971
13972
13973 // Set the nodes to be at the same positions
13974 // as the original just in case the
13975 // triangulatio is out of sync with the real data
13976 const unsigned n_node = original_mesh_pt->nnode();
13977 for (unsigned n = 0; n < n_node; ++n)
13978 {
13980 Node* const new_node_pt = tmesh_pt->node_pt(n);
13981 unsigned n_dim = nod_pt->ndim();
13982 for (unsigned i = 0; i < n_dim; ++i)
13983 {
13984 new_node_pt->x(i) = nod_pt->x(i);
13985 }
13986 }
13987 }
13988 else
13989 {
13991 << "Info/warning: Original Mesh is not TriangleBased\n"
13992 << "... but the copy is!" << std::endl;
13993 }
13994 }
13995 else
13996 {
13997 oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
13998 << std::endl;
13999 }
14000 } // End of single mesh case
14001 // Otherwise loop over the submeshes
14002 else
14003 {
14004 for (unsigned m = 0; m < N_mesh; m++)
14005 {
14006 // Can we refine the submesh
14008 dynamic_cast<TreeBasedRefineableMeshBase*>(
14010 {
14011 // Is the adapt flag set
14012 if (mmesh_pt->is_adaptation_enabled())
14013 {
14014 // Now get the original problem's mesh
14016 dynamic_cast<TreeBasedRefineableMeshBase*>(
14017 this->mesh_pt(m)))
14018 {
14019 if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
14020 {
14022 << "Info/Warning: Adaptive Continuation is broken in "
14023 << "SolidElement" << std::endl;
14024 }
14025
14026 mmesh_pt->refine_base_mesh_as_in_reference_mesh(
14028 }
14029 else
14030 {
14031 oomph_info << "Info/Warning: Mesh in orginal problem is "
14032 "not refineable."
14033 << std::endl;
14034 }
14035 }
14036 else
14037 {
14039 << "Info/Warning: Mesh adaptation is disabled in copy."
14040 << std::endl;
14041 }
14042 }
14043 else if (TriangleMeshBase* tmesh_pt =
14044 dynamic_cast<TriangleMeshBase*>(
14046 {
14048 dynamic_cast<TriangleMeshBase*>(this->mesh_pt(m)))
14049 {
14050 if (dynamic_cast<SolidMesh*>(original_mesh_pt) != 0)
14051 {
14053 << "Info/Warning: Adaptive Continuation is broken in "
14054 << "SolidElement" << std::endl;
14055 }
14056
14057 // Remesh using the triangulateIO of the base mesh
14058 // Done via a file, so a bit hacky but this will be
14059 // superseded very soon
14060 std::ofstream tri_dump("triangle_mesh.dmp");
14061 original_mesh_pt->dump_triangulateio(tri_dump);
14062 tri_dump.close();
14063 std::ifstream tri_read("triangle_mesh.dmp");
14064 tmesh_pt->remesh_from_triangulateio(tri_read);
14065 tri_read.close();
14066
14067 // Set the nodes to be at the same positions
14068 // as the original just in case the
14069 // triangulatio is out of sync with the real data
14070 const unsigned n_node = original_mesh_pt->nnode();
14071 for (unsigned n = 0; n < n_node; ++n)
14072 {
14074 Node* const new_node_pt = tmesh_pt->node_pt(n);
14075 unsigned n_dim = nod_pt->ndim();
14076 for (unsigned i = 0; i < n_dim; ++i)
14077 {
14078 new_node_pt->x(i) = nod_pt->x(i);
14079 }
14080 }
14081 }
14082 else
14083 {
14085 << "Info/warning: Original Mesh is not TriangleBased\n"
14086 << "... but the copy is!" << std::endl;
14087 }
14088 }
14089 else
14090 {
14091 oomph_info << "Info/Warning: Mesh cannot be adapted in copy."
14092 << std::endl;
14093 }
14094 }
14095
14096
14097 // Must call actions after adapt
14098 Copy_of_problem_pt[c]->actions_after_adapt();
14099
14100 // rebuild the global mesh in the copy
14101 Copy_of_problem_pt[c]->rebuild_global_mesh();
14102
14103 } // End of multiple mesh case
14104
14105 // Must call actions after adapt
14106 Copy_of_problem_pt[c]->actions_after_adapt();
14107
14108 // Assign the equation numbers to the copy (quietly)
14110 }
14111
14112 // Check that the dofs match for each copy
14113#ifdef PARANOID
14114 // If the problems don't match then complain
14115 if (Copy_of_problem_pt[c]->ndof() != this->ndof())
14116 {
14117 std::ostringstream error_stream;
14118 error_stream << "Number of unknowns in the problem copy " << c << " "
14119 << "not equal to number in the original:\n"
14120 << this->ndof() << " (original) "
14121 << Copy_of_problem_pt[c]->ndof() << " (copy)\n";
14122
14123 throw OomphLibError(error_stream.str(),
14126 }
14127#endif
14128 }
14129
14130 // Need to set the Dof derivatives to the copied problem
14131 // Assign the eigenfunction(s) to the copied problems
14133 for (unsigned i = 0; i < ndof_local; i++)
14134 {
14135 Copy_of_problem_pt[0]->dof(i) = this->dof_derivative(i);
14136 Copy_of_problem_pt[1]->dof(i) = this->dof_current(i);
14137 }
14138 // Set all pinned values to zero
14139 Copy_of_problem_pt[0]->set_pinned_values_to_zero();
14140 // Don't need to for the current dofs that are actuall the dofs
14141
14142 // Now adapt
14144 this->get_all_error_estimates(base_error);
14146 Copy_of_problem_pt[0]->adapt_based_on_error_estimates(
14148 Copy_of_problem_pt[1]->adapt_based_on_error_estimates(
14150
14151 // Now sort out the Dof pointer
14153 if (Dof_derivative.size() != ndof_local)
14154 {
14155 Dof_derivative.resize(ndof_local, 0.0);
14156 }
14157 if (Dof_current.size() != ndof_local)
14158 {
14159 Dof_current.resize(ndof_local, 0.0);
14160 }
14161 for (unsigned i = 0; i < ndof_local; i++)
14162 {
14164 Dof_current[i] = Copy_of_problem_pt[1]->dof(i);
14165 }
14166 // Return immediately
14167 return;
14168 }
14169
14170 oomph_info << std::endl << std::endl;
14171 oomph_info << "Adapting problem:" << std::endl;
14172 oomph_info << "=================" << std::endl;
14173
14174 double t_start = 0.0;
14176 {
14178 }
14179
14180 // Call the actions before adaptation
14182
14183 double t_end = 0.0;
14185 {
14187 oomph_info << "Time for actions before adapt: " << t_end - t_start
14188 << std::endl;
14190 }
14191
14192 // Initialise counters
14193 n_refined = 0;
14194 n_unrefined = 0;
14195
14196 // Number of submeshes?
14197 unsigned Nmesh = nsub_mesh();
14198
14199 // Single mesh:
14200 //------------
14201 if (Nmesh == 0)
14202 {
14203 // Refine single mesh if possible
14205 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14206 {
14207 if (mmesh_pt->is_adaptation_enabled())
14208 {
14209 double t_start = TimingHelpers::timer();
14210
14211 // Get pointer to error estimator
14213 mmesh_pt->spatial_error_estimator_pt();
14214
14215#ifdef PARANOID
14216 if (error_estimator_pt == 0)
14217 {
14218 throw OomphLibError("Error estimator hasn't been set yet",
14221 }
14222#endif
14223
14224 // Get error for all elements
14226
14227 if (mmesh_pt->doc_info_pt() == 0)
14228 {
14229 error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14230 }
14231 else
14232 {
14233 error_estimator_pt->get_element_errors(
14234 mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14235 }
14236
14237 // Store max./min actual error
14238 mmesh_pt->max_error() = std::fabs(*std::max_element(
14240
14241 mmesh_pt->min_error() = std::fabs(*std::min_element(
14243
14244 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14245 << mmesh_pt->min_error() << std::endl
14246 << std::endl;
14247
14248
14250 {
14252 oomph_info << "Time for error estimation: " << t_end - t_start
14253 << std::endl;
14255 }
14256
14257 // Adapt mesh
14258 mmesh_pt->adapt(elemental_error);
14259
14260 // Add to counters
14261 n_refined += mmesh_pt->nrefined();
14262 n_unrefined += mmesh_pt->nunrefined();
14263
14265 {
14267 oomph_info << "Time for complete mesh adaptation "
14268 << "(but excluding comp of error estimate): "
14269 << t_end - t_start << std::endl;
14271 }
14272 }
14273 else
14274 {
14275 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14276 << std::endl;
14277 }
14278 }
14279 else
14280 {
14281 oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14282 }
14283 }
14284 // Multiple submeshes
14285 //------------------
14286 else
14287 {
14288 // Loop over submeshes
14289 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14290 {
14291 // Refine single mesh uniformly if possible
14293 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14294 {
14295 double t_start = TimingHelpers::timer();
14296
14297 // Get pointer to error estimator
14299 mmesh_pt->spatial_error_estimator_pt();
14300
14301#ifdef PARANOID
14302 if (error_estimator_pt == 0)
14303 {
14304 throw OomphLibError("Error estimator hasn't been set yet",
14307 }
14308#endif
14309
14310 if (mmesh_pt->is_adaptation_enabled())
14311 {
14312 // Get error for all elements
14314 if (mmesh_pt->doc_info_pt() == 0)
14315 {
14316 error_estimator_pt->get_element_errors(mesh_pt(imesh),
14318 }
14319 else
14320 {
14321 error_estimator_pt->get_element_errors(
14322 mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14323 }
14324
14325 // Store max./min error if the mesh has any elements
14326 if (mesh_pt(imesh)->nelement() > 0)
14327 {
14328 mmesh_pt->max_error() =
14329 std::fabs(*std::max_element(elemental_error.begin(),
14330 elemental_error.end(),
14331 AbsCmp<double>()));
14332
14333 mmesh_pt->min_error() =
14334 std::fabs(*std::min_element(elemental_error.begin(),
14335 elemental_error.end(),
14336 AbsCmp<double>()));
14337 }
14338
14339 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14340 << mmesh_pt->min_error() << std::endl;
14341
14342
14344 {
14346 oomph_info << "Time for error estimation: " << t_end - t_start
14347 << std::endl;
14349 }
14350
14351 // Adapt mesh
14352 mmesh_pt->adapt(elemental_error);
14353
14354 // Add to counters
14355 n_refined += mmesh_pt->nrefined();
14356 n_unrefined += mmesh_pt->nunrefined();
14357
14358
14360 {
14362 oomph_info << "Time for complete mesh adaptation "
14363 << "(but excluding comp of error estimate): "
14364 << t_end - t_start << std::endl;
14366 }
14367 }
14368 else
14369 {
14370 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14371 << std::endl;
14372 }
14373 }
14374 else
14375 {
14376 oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14377 }
14378
14379 } // End of loop over submeshes
14380
14381 // Rebuild the global mesh
14383 }
14384
14385
14387 {
14389 oomph_info << "Total time for actual adaptation "
14390 << "(all meshes; incl error estimates): " << t_end - t_start
14391 << std::endl;
14393 }
14394
14395 // Any actions after adapt
14397
14398
14400 {
14402 oomph_info << "Time for actions after adapt: " << t_end - t_start
14403 << std::endl;
14405
14406 oomph_info << "About to start re-assigning eqn numbers "
14407 << "with Problem::assign_eqn_numbers() at end of "
14408 << "Problem::adapt().\n";
14409 }
14410
14411 // Attach the boundary conditions to the mesh
14412 oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14413 << std::endl;
14414
14415
14417 {
14419 oomph_info << "Time for re-assigning eqn numbers with "
14420 << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14421 << t_end - t_start << std::endl;
14422 oomph_info << "Total time for adapt: " << t_end - t_start_total
14423 << std::endl;
14424 }
14425 }
14426
14427 //========================================================================
14428 /// p-adapt problem:
14429 /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14430 /// based on their own error estimates and the target errors specified
14431 /// in the mesh(es). Following mesh adaptation,
14432 /// update global mesh, and re-assign equation numbers.
14433 /// Return # of refined/unrefined elements. On return from this
14434 /// function, Problem can immediately be solved again.
14435 //======================================================================
14436 void Problem::p_adapt(unsigned& n_refined, unsigned& n_unrefined)
14437 {
14438 double t_start_total = 0.0;
14440 {
14442 }
14443
14444 // Get the bifurcation type
14445 int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14446
14447 // If we are tracking a bifurcation then call the bifurcation adapt function
14448 if (bifurcation_type != 0)
14449 {
14450 this->bifurcation_adapt_helper(n_refined, n_unrefined, bifurcation_type);
14451 // Return immediately
14452 return;
14453 }
14454
14455 oomph_info << std::endl << std::endl;
14456 oomph_info << "p-adapting problem:" << std::endl;
14457 oomph_info << "===================" << std::endl;
14458
14459 double t_start = 0.0;
14461 {
14463 }
14464
14465 // Call the actions before adaptation
14467
14468 double t_end = 0.0;
14470 {
14472 oomph_info << "Time for actions before adapt: " << t_end - t_start
14473 << std::endl;
14475 }
14476
14477 // Initialise counters
14478 n_refined = 0;
14479 n_unrefined = 0;
14480
14481 // Number of submeshes?
14482 unsigned Nmesh = nsub_mesh();
14483
14484 // Single mesh:
14485 //------------
14486 if (Nmesh == 0)
14487 {
14488 // Refine single mesh if possible
14490 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14491 {
14492 if (mmesh_pt->is_p_adaptation_enabled())
14493 {
14494 double t_start = TimingHelpers::timer();
14495
14496 // Get pointer to error estimator
14498 mmesh_pt->spatial_error_estimator_pt();
14499
14500#ifdef PARANOID
14501 if (error_estimator_pt == 0)
14502 {
14503 throw OomphLibError("Error estimator hasn't been set yet",
14506 }
14507#endif
14508
14509 // Get error for all elements
14511
14512 if (mmesh_pt->doc_info_pt() == 0)
14513 {
14514 error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
14515 }
14516 else
14517 {
14518 error_estimator_pt->get_element_errors(
14519 mesh_pt(0), elemental_error, *mmesh_pt->doc_info_pt());
14520 }
14521
14522 // Store max./min actual error
14523 mmesh_pt->max_error() = std::fabs(*std::max_element(
14525
14526 mmesh_pt->min_error() = std::fabs(*std::min_element(
14528
14529 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14530 << mmesh_pt->min_error() << std::endl
14531 << std::endl;
14532
14533
14535 {
14537 oomph_info << "Time for error estimation: " << t_end - t_start
14538 << std::endl;
14540 }
14541
14542 // Adapt mesh
14543 mmesh_pt->p_adapt(elemental_error);
14544
14545 // Add to counters
14546 n_refined += mmesh_pt->nrefined();
14547 n_unrefined += mmesh_pt->nunrefined();
14548
14550 {
14552 oomph_info << "Time for complete mesh adaptation "
14553 << "(but excluding comp of error estimate): "
14554 << t_end - t_start << std::endl;
14556 }
14557 }
14558 else
14559 {
14560 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14561 << std::endl;
14562 }
14563 }
14564 else
14565 {
14566 oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14567 }
14568 }
14569 // Multiple submeshes
14570 //------------------
14571 else
14572 {
14573 // Loop over submeshes
14574 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14575 {
14576 // Refine single mesh uniformly if possible
14578 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
14579 {
14580 double t_start = TimingHelpers::timer();
14581
14582 // Get pointer to error estimator
14584 mmesh_pt->spatial_error_estimator_pt();
14585
14586#ifdef PARANOID
14587 if (error_estimator_pt == 0)
14588 {
14589 throw OomphLibError("Error estimator hasn't been set yet",
14592 }
14593#endif
14594
14595 if (mmesh_pt->is_p_adaptation_enabled())
14596 {
14597 // Get error for all elements
14599 if (mmesh_pt->doc_info_pt() == 0)
14600 {
14601 error_estimator_pt->get_element_errors(mesh_pt(imesh),
14603 }
14604 else
14605 {
14606 error_estimator_pt->get_element_errors(
14607 mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
14608 }
14609
14610 // Store max./min error if the mesh has any elements
14611 if (mesh_pt(imesh)->nelement() > 0)
14612 {
14613 mmesh_pt->max_error() =
14614 std::fabs(*std::max_element(elemental_error.begin(),
14615 elemental_error.end(),
14616 AbsCmp<double>()));
14617
14618 mmesh_pt->min_error() =
14619 std::fabs(*std::min_element(elemental_error.begin(),
14620 elemental_error.end(),
14621 AbsCmp<double>()));
14622 }
14623
14624 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14625 << mmesh_pt->min_error() << std::endl;
14626
14627
14629 {
14631 oomph_info << "Time for error estimation: " << t_end - t_start
14632 << std::endl;
14634 }
14635
14636 // Adapt mesh
14637 mmesh_pt->p_adapt(elemental_error);
14638
14639 // Add to counters
14640 n_refined += mmesh_pt->nrefined();
14641 n_unrefined += mmesh_pt->nunrefined();
14642
14643
14645 {
14647 oomph_info << "Time for complete mesh adaptation "
14648 << "(but excluding comp of error estimate): "
14649 << t_end - t_start << std::endl;
14651 }
14652 }
14653 else
14654 {
14655 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14656 << std::endl;
14657 }
14658 }
14659 else
14660 {
14661 oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14662 }
14663
14664 } // End of loop over submeshes
14665
14666 // Rebuild the global mesh
14668 }
14669
14670
14672 {
14674 oomph_info << "Total time for actual adaptation "
14675 << "(all meshes; incl error estimates): " << t_end - t_start
14676 << std::endl;
14678 }
14679
14680 // Any actions after adapt
14682
14683
14685 {
14687 oomph_info << "Time for actions after adapt: " << t_end - t_start
14688 << std::endl;
14690
14691 oomph_info << "About to start re-assigning eqn numbers "
14692 << "with Problem::assign_eqn_numbers() at end of "
14693 << "Problem::adapt().\n";
14694 }
14695
14696 // Attach the boundary conditions to the mesh
14697 oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14698 << std::endl;
14699
14700
14702 {
14704 oomph_info << "Time for re-assigning eqn numbers with "
14705 << "Problem::assign_eqn_numbers() at end of Problem::adapt(): "
14706 << t_end - t_start << std::endl;
14707 oomph_info << "Total time for adapt: " << t_end - t_start_total
14708 << std::endl;
14709 }
14710 }
14711
14712 //========================================================================
14713 /// Perform mesh adaptation for (all) refineable (sub)mesh(es),
14714 /// based on the error estimates in elemental_error
14715 /// and the target errors specified
14716 /// in the mesh(es). Following mesh adaptation,
14717 /// update global mesh, and re-assign equation numbers.
14718 /// Return # of refined/unrefined elements. On return from this
14719 /// function, Problem can immediately be solved again.
14720 //========================================================================
14722 unsigned& n_refined,
14723 unsigned& n_unrefined,
14725 {
14726 oomph_info << std::endl << std::endl;
14727 oomph_info << "Adapting problem:" << std::endl;
14728 oomph_info << "=================" << std::endl;
14729
14730 // Call the actions before adaptation
14732
14733 // Initialise counters
14734 n_refined = 0;
14735 n_unrefined = 0;
14736
14737 // Number of submeshes?
14738 unsigned Nmesh = nsub_mesh();
14739
14740 // Single mesh:
14741 //------------
14742 if (Nmesh == 0)
14743 {
14744 // Refine single mesh uniformly if possible
14746 dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14747 {
14748 if (mmesh_pt->is_adaptation_enabled())
14749 {
14750 // Adapt mesh
14751 mmesh_pt->adapt(elemental_error[0]);
14752
14753 // Add to counters
14754 n_refined += mmesh_pt->nrefined();
14755 n_unrefined += mmesh_pt->nunrefined();
14756 }
14757 else
14758 {
14759 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14760 << std::endl;
14761 }
14762 }
14763 else
14764 {
14765 oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14766 }
14767 }
14768
14769 // Multiple submeshes
14770 //------------------
14771 else
14772 {
14773 // Loop over submeshes
14774 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14775 {
14776 // Refine single mesh uniformly if possible
14778 dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14779 {
14780 if (mmesh_pt->is_adaptation_enabled())
14781 {
14782 // Adapt mesh
14784
14785 // Add to counters
14786 n_refined += mmesh_pt->nrefined();
14787 n_unrefined += mmesh_pt->nunrefined();
14788 }
14789 else
14790 {
14791 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14792 << std::endl;
14793 }
14794 }
14795 else
14796 {
14797 oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14798 }
14799
14800 } // End of loop over submeshes
14801
14802 // Rebuild the global mesh
14804 }
14805
14806 // Any actions after adapt
14808
14809 // Attach the boundary conditions to the mesh
14810 oomph_info << "\nNumber of equations: " << assign_eqn_numbers() << std::endl
14811 << std::endl;
14812 }
14813
14814
14815 //========================================================================
14816 /// Return the error estimates computed by (all) refineable
14817 /// (sub)mesh(es) in the elemental_error structure, which consists of
14818 /// a vector of elemental errors for each (sub)mesh.
14819 //========================================================================
14821 {
14822 // Number of submeshes?
14823 const unsigned Nmesh = nsub_mesh();
14824
14825 // Single mesh:
14826 //------------
14827 if (Nmesh == 0)
14828 {
14829 // There is only one mesh
14830 elemental_error.resize(1);
14831 // Refine single mesh uniformly if possible
14833 dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(0)))
14834 {
14835 // If we can adapt the mesh
14836 if (mmesh_pt->is_adaptation_enabled())
14837 {
14838 // Get pointer to error estimator
14840 mmesh_pt->spatial_error_estimator_pt();
14841
14842#ifdef PARANOID
14843 if (error_estimator_pt == 0)
14844 {
14845 throw OomphLibError("Error estimator hasn't been set yet",
14848 }
14849#endif
14850
14851 // Get error for all elements
14852 elemental_error[0].resize(mmesh_pt->nelement());
14853 // Are we documenting the errors or not
14854 if (mmesh_pt->doc_info_pt() == 0)
14855 {
14856 error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14857 elemental_error[0]);
14858 }
14859 else
14860 {
14861 error_estimator_pt->get_element_errors(Problem::mesh_pt(0),
14862 elemental_error[0],
14863 *mmesh_pt->doc_info_pt());
14864 }
14865
14866 // Store max./min actual error
14867 mmesh_pt->max_error() =
14868 std::fabs(*std::max_element(elemental_error[0].begin(),
14869 elemental_error[0].end(),
14870 AbsCmp<double>()));
14871
14872 mmesh_pt->min_error() =
14873 std::fabs(*std::min_element(elemental_error[0].begin(),
14874 elemental_error[0].end(),
14875 AbsCmp<double>()));
14876
14877 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14878 << mmesh_pt->min_error() << std::endl;
14879 }
14880 else
14881 {
14882 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14883 << std::endl;
14884 }
14885 }
14886 else
14887 {
14888 oomph_info << "Info/Warning: Mesh cannot be adapted" << std::endl;
14889 }
14890 }
14891
14892 // Multiple submeshes
14893 //------------------
14894 else
14895 {
14896 // Resize to the number of submeshes
14897 elemental_error.resize(Nmesh);
14898
14899 // Loop over submeshes
14900 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
14901 {
14902 // Refine single mesh uniformly if possible
14904 dynamic_cast<RefineableMeshBase*>(Problem::mesh_pt(imesh)))
14905 {
14906 // Get pointer to error estimator
14908 mmesh_pt->spatial_error_estimator_pt();
14909
14910#ifdef PARANOID
14911 if (error_estimator_pt == 0)
14912 {
14913 throw OomphLibError("Error estimator hasn't been set yet",
14916 }
14917#endif
14918 // If we can adapt the mesh
14919 if (mmesh_pt->is_adaptation_enabled())
14920 {
14921 // Get error for all elements
14922 elemental_error[imesh].resize(mmesh_pt->nelement());
14923 if (mmesh_pt->doc_info_pt() == 0)
14924 {
14925 error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14927 }
14928 else
14929 {
14930 error_estimator_pt->get_element_errors(Problem::mesh_pt(imesh),
14932 *mmesh_pt->doc_info_pt());
14933 }
14934
14935 // Store max./min error
14936 mmesh_pt->max_error() =
14937 std::fabs(*std::max_element(elemental_error[imesh].begin(),
14939 AbsCmp<double>()));
14940
14941 mmesh_pt->min_error() =
14942 std::fabs(*std::min_element(elemental_error[imesh].begin(),
14944 AbsCmp<double>()));
14945
14946 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
14947 << mmesh_pt->min_error() << std::endl;
14948 }
14949 else
14950 {
14951 oomph_info << "Info/Warning: Mesh adaptation is disabled."
14952 << std::endl;
14953 }
14954 }
14955 else
14956 {
14957 oomph_info << "Info/Warning: Mesh cannot be adapted." << std::endl;
14958 }
14959
14960 } // End of loop over submeshes
14961 }
14962 }
14963
14964 //========================================================================
14965 /// Get max and min error for all elements in submeshes
14966 //========================================================================
14968 {
14969 // Get the bifurcation type
14970 int bifurcation_type = this->Assembly_handler_pt->bifurcation_type();
14971 // If we are tracking a bifurcation then call the bifurcation adapt function
14972 if (bifurcation_type != 0)
14973 {
14974 this->bifurcation_adapt_doc_errors(bifurcation_type);
14975 // Return immediately
14976 return;
14977 }
14978
14979 // Number of submeshes?
14980 unsigned Nmesh = nsub_mesh();
14981
14982 // Single mesh:
14983 //------------
14984 if (Nmesh == 0)
14985 {
14986 // Is the single mesh refineable?
14988 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
14989 {
14990 // Get pointer to error estimator
14992 mmesh_pt->spatial_error_estimator_pt();
14993
14994#ifdef PARANOID
14995 if (error_estimator_pt == 0)
14996 {
14997 throw OomphLibError("Error estimator hasn't been set yet",
15000 }
15001#endif
15002
15003 // Get error for all elements
15005 if (!doc_info.is_doc_enabled())
15006 {
15007 error_estimator_pt->get_element_errors(mesh_pt(0), elemental_error);
15008 }
15009 else
15010 {
15011 error_estimator_pt->get_element_errors(
15012 mesh_pt(0), elemental_error, doc_info);
15013 }
15014
15015 // Store max./min actual error
15016 mmesh_pt->max_error() = std::fabs(*std::max_element(
15018
15019 mmesh_pt->min_error() = std::fabs(*std::min_element(
15021
15022 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
15023 << mmesh_pt->min_error() << std::endl;
15024 }
15025 }
15026
15027 // Multiple submeshes
15028 //------------------
15029 else
15030 {
15031 // Loop over submeshes
15032 for (unsigned imesh = 0; imesh < Nmesh; imesh++)
15033 {
15034 // Is the single mesh refineable?
15036 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15037 {
15038 // Get pointer to error estimator
15040 mmesh_pt->spatial_error_estimator_pt();
15041
15042#ifdef PARANOID
15043 if (error_estimator_pt == 0)
15044 {
15045 throw OomphLibError("Error estimator hasn't been set yet",
15048 }
15049#endif
15050
15051 // Get error for all elements
15053 if (mmesh_pt->doc_info_pt() == 0)
15054 {
15055 error_estimator_pt->get_element_errors(mesh_pt(imesh),
15057 }
15058 else
15059 {
15060 error_estimator_pt->get_element_errors(
15061 mesh_pt(imesh), elemental_error, *mmesh_pt->doc_info_pt());
15062 }
15063
15064 // Store max./min error if the mesh has any elements
15065 if (mesh_pt(imesh)->nelement() > 0)
15066 {
15067 mmesh_pt->max_error() =
15068 std::fabs(*std::max_element(elemental_error.begin(),
15069 elemental_error.end(),
15070 AbsCmp<double>()));
15071
15072 mmesh_pt->min_error() =
15073 std::fabs(*std::min_element(elemental_error.begin(),
15074 elemental_error.end(),
15075 AbsCmp<double>()));
15076 }
15077
15078 oomph_info << "\n Max/min error: " << mmesh_pt->max_error() << " "
15079 << mmesh_pt->min_error() << std::endl;
15080 }
15081
15082 } // End of loop over submeshes
15083 }
15084 }
15085
15086 //========================================================================
15087 /// Refine (one and only!) mesh by splitting the elements identified
15088 /// by their numbers relative to the problems' only mesh, then rebuild
15089 /// the problem.
15090 //========================================================================
15093 {
15095
15096 // Number of submeshes?
15097 unsigned Nmesh = nsub_mesh();
15098
15099 // Single mesh:
15100 if (Nmesh == 0)
15101 {
15102 // Refine single mesh if possible
15104 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15105 {
15106 mmesh_pt->refine_selected_elements(elements_to_be_refined);
15107 }
15108 else
15109 {
15110 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15111 }
15112 }
15113 // Multiple submeshes
15114 else
15115 {
15116 std::ostringstream error_message;
15117 error_message << "Problem::refine_selected_elements(...) only works for\n"
15118 << "multiple-mesh problems if you specify the mesh\n"
15119 << "number in the function argument before the Vector,\n"
15120 << "or a Vector of Vectors for each submesh.\n"
15121 << std::endl;
15122 throw OomphLibError(
15123 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15124 }
15125
15126 // Any actions after the adapatation phase
15128
15129 // Attach the boundary conditions to the mesh
15130 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15131 }
15132
15133 //========================================================================
15134 /// Refine (one and only!) mesh by splitting the elements identified
15135 /// by their pointers, then rebuild the problem.
15136 //========================================================================
15139 {
15141
15142 // Number of submeshes?
15143 unsigned Nmesh = nsub_mesh();
15144
15145 // Single mesh:
15146 if (Nmesh == 0)
15147 {
15148 // Refine single mesh if possible
15150 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15151 {
15152 mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15153 }
15154 else
15155 {
15156 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15157 }
15158 }
15159 // Multiple submeshes
15160 else
15161 {
15162 std::ostringstream error_message;
15163 error_message << "Problem::refine_selected_elements(...) only works for\n"
15164 << "multiple-mesh problems if you specify the mesh\n"
15165 << "number in the function argument before the Vector,\n"
15166 << "or a Vector of Vectors for each submesh.\n"
15167 << std::endl;
15168 throw OomphLibError(
15169 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15170 }
15171
15172 // Any actions after the adapatation phase
15174
15175 // Do equation numbering
15176 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15177 }
15178
15179 //========================================================================
15180 /// Refine specified submesh by splitting the elements identified
15181 /// by their numbers relative to the specified mesh, then rebuild the problem.
15182 //========================================================================
15184 const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15185 {
15187
15188 // Number of submeshes?
15189 unsigned n_mesh = nsub_mesh();
15190
15191 if (i_mesh >= n_mesh)
15192 {
15193 std::ostringstream error_message;
15194 error_message << "Problem only has " << n_mesh
15195 << " submeshes. Cannot refine submesh " << i_mesh
15196 << std::endl;
15197 throw OomphLibError(
15198 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15199 }
15200
15201 // Refine single mesh if possible
15204 {
15205 mmesh_pt->refine_selected_elements(elements_to_be_refined);
15206 }
15207 else
15208 {
15209 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15210 }
15211
15212 if (n_mesh > 1)
15213 {
15214 // Rebuild the global mesh
15216 }
15217
15218 // Any actions after the adapatation phase
15220
15221 // Do equation numbering
15222 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15223 }
15224
15225
15226 //========================================================================
15227 /// Refine specified submesh by splitting the elements identified
15228 /// by their pointers, then rebuild the problem.
15229 //========================================================================
15231 const unsigned& i_mesh,
15233 {
15235
15236 // Number of submeshes?
15237 unsigned n_mesh = nsub_mesh();
15238
15239 if (i_mesh >= n_mesh)
15240 {
15241 std::ostringstream error_message;
15242 error_message << "Problem only has " << n_mesh
15243 << " submeshes. Cannot refine submesh " << i_mesh
15244 << std::endl;
15245 throw OomphLibError(
15246 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15247 }
15248
15249 // Refine single mesh if possible
15252 {
15253 mmesh_pt->refine_selected_elements(elements_to_be_refined_pt);
15254 }
15255 else
15256 {
15257 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15258 }
15259
15260 if (n_mesh > 1)
15261 {
15262 // Rebuild the global mesh
15264 }
15265
15266 // Any actions after the adapatation phase
15268
15269 // Do equation numbering
15270 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15271 }
15272
15273 //========================================================================
15274 /// Refine all submeshes by splitting the elements identified by their
15275 /// numbers relative to each submesh in a Vector of Vectors, then
15276 /// rebuild the problem.
15277 //========================================================================
15280 {
15282
15283 // Number of submeshes?
15284 unsigned n_mesh = nsub_mesh();
15285
15286 // Refine all submeshes if possible
15287 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15288 {
15291 {
15292 mmesh_pt->refine_selected_elements(elements_to_be_refined[i_mesh]);
15293 }
15294 else
15295 {
15296 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15297 }
15298 }
15299
15300 // Rebuild the global mesh
15302
15303 // Any actions after the adapatation phase
15305
15306 // Do equation numbering
15307 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15308 }
15309
15310 //========================================================================
15311 /// Refine all submeshes by splitting the elements identified by their
15312 /// pointers within each submesh in a Vector of Vectors, then
15313 /// rebuild the problem.
15314 //========================================================================
15317 {
15319
15320 // Number of submeshes?
15321 unsigned n_mesh = nsub_mesh();
15322
15323 // Refine all submeshes if possible
15324 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15325 {
15328 {
15329 mmesh_pt->refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15330 }
15331 else
15332 {
15333 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15334 }
15335 }
15336
15337 // Rebuild the global mesh
15339
15340 // Any actions after the adapatation phase
15342
15343 // Do equation numbering
15344 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15345 }
15346
15347 //========================================================================
15348 /// p-refine (one and only!) mesh by refining the elements identified
15349 /// by their numbers relative to the problems' only mesh, then rebuild
15350 /// the problem.
15351 //========================================================================
15354 {
15356
15357 // Number of submeshes?
15358 unsigned Nmesh = nsub_mesh();
15359
15360 // Single mesh:
15361 if (Nmesh == 0)
15362 {
15363 // Refine single mesh if possible
15365 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15366 {
15367 mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15368 }
15369 else
15370 {
15371 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15372 }
15373 }
15374 // Multiple submeshes
15375 else
15376 {
15377 std::ostringstream error_message;
15378 error_message
15379 << "Problem::p_refine_selected_elements(...) only works for\n"
15380 << "multiple-mesh problems if you specify the mesh\n"
15381 << "number in the function argument before the Vector,\n"
15382 << "or a Vector of Vectors for each submesh.\n"
15383 << std::endl;
15384 throw OomphLibError(
15385 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15386 }
15387
15388 // Any actions after the adapatation phase
15390
15391 // Attach the boundary conditions to the mesh
15392 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15393 }
15394
15395 //========================================================================
15396 /// p-refine (one and only!) mesh by refining the elements identified
15397 /// by their pointers, then rebuild the problem.
15398 //========================================================================
15401 {
15403
15404 // Number of submeshes?
15405 unsigned Nmesh = nsub_mesh();
15406
15407 // Single mesh:
15408 if (Nmesh == 0)
15409 {
15410 // Refine single mesh if possible
15412 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt(0)))
15413 {
15414 mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15415 }
15416 else
15417 {
15418 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15419 }
15420 }
15421 // Multiple submeshes
15422 else
15423 {
15424 std::ostringstream error_message;
15425 error_message
15426 << "Problem::p_refine_selected_elements(...) only works for\n"
15427 << "multiple-mesh problems if you specify the mesh\n"
15428 << "number in the function argument before the Vector,\n"
15429 << "or a Vector of Vectors for each submesh.\n"
15430 << std::endl;
15431 throw OomphLibError(
15432 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15433 }
15434
15435 // Any actions after the adapatation phase
15437
15438 // Do equation numbering
15439 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15440 }
15441
15442 //========================================================================
15443 /// p-refine specified submesh by refining the elements identified
15444 /// by their numbers relative to the specified mesh, then rebuild the problem.
15445 //========================================================================
15447 const unsigned& i_mesh, const Vector<unsigned>& elements_to_be_refined)
15448 {
15450 "p-refinement for multiple submeshes has not yet been tested.",
15451 "Problem::p_refine_selected_elements()",
15453
15455
15456 // Number of submeshes?
15457 unsigned n_mesh = nsub_mesh();
15458
15459 if (i_mesh >= n_mesh)
15460 {
15461 std::ostringstream error_message;
15462 error_message << "Problem only has " << n_mesh
15463 << " submeshes. Cannot p-refine submesh " << i_mesh
15464 << std::endl;
15465 throw OomphLibError(
15466 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15467 }
15468
15469 // Refine single mesh if possible
15472 {
15473 mmesh_pt->p_refine_selected_elements(elements_to_be_refined);
15474 }
15475 else
15476 {
15477 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15478 }
15479
15480 if (n_mesh > 1)
15481 {
15482 // Rebuild the global mesh
15484 }
15485
15486 // Any actions after the adapatation phase
15488
15489 // Do equation numbering
15490 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15491 }
15492
15493
15494 //========================================================================
15495 /// p-refine specified submesh by refining the elements identified
15496 /// by their pointers, then rebuild the problem.
15497 //========================================================================
15499 const unsigned& i_mesh,
15501 {
15503 "p-refinement for multiple submeshes has not yet been tested.",
15504 "Problem::p_refine_selected_elements()",
15506
15508
15509 // Number of submeshes?
15510 unsigned n_mesh = nsub_mesh();
15511
15512 if (i_mesh >= n_mesh)
15513 {
15514 std::ostringstream error_message;
15515 error_message << "Problem only has " << n_mesh
15516 << " submeshes. Cannot p-refine submesh " << i_mesh
15517 << std::endl;
15518 throw OomphLibError(
15519 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15520 }
15521
15522 // Refine single mesh if possible
15525 {
15526 mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt);
15527 }
15528 else
15529 {
15530 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15531 }
15532
15533 if (n_mesh > 1)
15534 {
15535 // Rebuild the global mesh
15537 }
15538
15539 // Any actions after the adapatation phase
15541
15542 // Do equation numbering
15543 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15544 }
15545
15546 //========================================================================
15547 /// p-refine all submeshes by refining the elements identified by their
15548 /// numbers relative to each submesh in a Vector of Vectors, then
15549 /// rebuild the problem.
15550 //========================================================================
15553 {
15555 "p-refinement for multiple submeshes has not yet been tested.",
15556 "Problem::p_refine_selected_elements()",
15558
15560
15561 // Number of submeshes?
15562 unsigned n_mesh = nsub_mesh();
15563
15564 // Refine all submeshes if possible
15565 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15566 {
15569 {
15570 mmesh_pt->p_refine_selected_elements(elements_to_be_refined[i_mesh]);
15571 }
15572 else
15573 {
15574 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15575 }
15576 }
15577
15578 // Rebuild the global mesh
15580
15581 // Any actions after the adapatation phase
15583
15584 // Do equation numbering
15585 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15586 }
15587
15588 //========================================================================
15589 /// p-refine all submeshes by refining the elements identified by their
15590 /// pointers within each submesh in a Vector of Vectors, then
15591 /// rebuild the problem.
15592 //========================================================================
15595 {
15597 "p-refinement for multiple submeshes has not yet been tested.",
15598 "Problem::p_refine_selected_elements()",
15600
15602
15603 // Number of submeshes?
15604 unsigned n_mesh = nsub_mesh();
15605
15606 // Refine all submeshes if possible
15607 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
15608 {
15611 {
15612 mmesh_pt->p_refine_selected_elements(elements_to_be_refined_pt[i_mesh]);
15613 }
15614 else
15615 {
15616 oomph_info << "Info/Warning: Mesh cannot be refined " << std::endl;
15617 }
15618 }
15619
15620 // Rebuild the global mesh
15622
15623 // Any actions after the adapatation phase
15625
15626 // Do equation numbering
15627 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15628 }
15629
15630
15631 //========================================================================
15632 /// Helper function to do compund refinement of (all) refineable
15633 /// (sub)mesh(es) uniformly as many times as specified in vector and
15634 /// rebuild problem; doc refinement process. Set boolean argument
15635 /// to true if you want to prune immediately after refining the meshes
15636 /// individually.
15637 //========================================================================
15639 DocInfo& doc_info,
15640 const bool& prune)
15641 {
15642 double t_start = 0.0;
15644 {
15646 }
15647
15649
15650 double t_end = 0.0;
15652 {
15655 << "Time for actions before adapt in Problem::refine_uniformly_aux(): "
15656 << t_end - t_start << std::endl;
15658 }
15659
15660 // Number of submeshes?
15661 unsigned n_mesh = nsub_mesh();
15662
15663 // Single mesh:
15664 if (n_mesh == 0)
15665 {
15666 // Refine single mesh uniformly if possible
15668 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15669 {
15670 unsigned nref = nrefine_for_mesh[0];
15671 for (unsigned i = 0; i < nref; i++)
15672 {
15673 mmesh_pt->refine_uniformly(doc_info);
15674 }
15675 }
15676 else
15677 {
15678 oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15679 << std::endl;
15680 }
15681 }
15682 // Multiple submeshes
15683 else
15684 {
15685 // Loop over submeshes
15686 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15687 {
15688 // Refine i-th submesh uniformly if possible
15690 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15691 {
15692 unsigned nref = nrefine_for_mesh[imesh];
15693 for (unsigned i = 0; i < nref; i++)
15694 {
15695 mmesh_pt->refine_uniformly(doc_info);
15696 }
15697 }
15698 else
15699 {
15700 oomph_info << "Info/Warning: Cannot refine mesh " << imesh
15701 << std::endl;
15702 }
15703 }
15704 // Rebuild the global mesh
15706 }
15707
15709 {
15711 oomph_info << "Time for mesh-level mesh refinement in "
15712 << "Problem::refine_uniformly_aux(): " << t_end - t_start
15713 << std::endl;
15715 }
15716
15717 // Any actions after the adaptation phase
15719
15720
15722 {
15725 << "Time for actions after adapt Problem::refine_uniformly_aux(): "
15726 << t_end - t_start << std::endl;
15728 }
15729
15730
15731#ifdef OOMPH_HAS_MPI
15732
15733 // Prune it?
15734 if (prune)
15735 {
15736 // Note: This calls assign eqn numbers already...
15740
15742 {
15744 oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15745 << "Problem::refine_uniformly_aux(): " << t_end - t_start
15746 << std::endl;
15747 }
15748 }
15749 else
15750#else
15751 if (prune)
15752 {
15753 std::ostringstream error_message;
15754 error_message
15755 << "Requested pruning in serial build. Ignoring the request.\n";
15756 OomphLibWarning(error_message.str(),
15757 "Problem::refine_uniformly_aux()",
15759 }
15760#endif
15761 {
15762 // Do equation numbering
15764 << "Number of equations after Problem::refine_uniformly_aux(): "
15765 << assign_eqn_numbers() << std::endl;
15766
15768 {
15770 oomph_info << "Time for Problem::assign_eqn_numbers() in "
15771 << "Problem::refine_uniformly_aux(): " << t_end - t_start
15772 << std::endl;
15773 }
15774 }
15775 }
15776
15777
15778 //========================================================================
15779 /// Helper function to do compund p-refinement of (all) p-refineable
15780 /// (sub)mesh(es) uniformly as many times as specified in vector and
15781 /// rebuild problem; doc refinement process. Set boolean argument
15782 /// to true if you want to prune immediately after refining the meshes
15783 /// individually.
15784 //========================================================================
15786 DocInfo& doc_info,
15787 const bool& prune)
15788 {
15789 double t_start = 0.0;
15791 {
15793 }
15794
15796
15797 double t_end = 0.0;
15799 {
15801 oomph_info << "Time for actions before adapt in "
15802 "Problem::p_refine_uniformly_aux(): "
15803 << t_end - t_start << std::endl;
15805 }
15806
15807 // Number of submeshes?
15808 unsigned n_mesh = nsub_mesh();
15809
15810 // Single mesh:
15811 if (n_mesh == 0)
15812 {
15813 // Refine single mesh uniformly if possible
15815 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
15816 {
15817 unsigned nref = nrefine_for_mesh[0];
15818 for (unsigned i = 0; i < nref; i++)
15819 {
15820 mmesh_pt->p_refine_uniformly(doc_info);
15821 }
15822 }
15823 else
15824 {
15825 oomph_info << "Info/Warning: Mesh cannot be p-refined uniformly "
15826 << std::endl;
15827 }
15828 }
15829 // Multiple submeshes
15830 else
15831 {
15833 "p-refinement for multiple submeshes has not yet been tested.",
15834 "Problem::p_refine_uniformly_aux()",
15836
15837 // Loop over submeshes
15838 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
15839 {
15840 // Refine i-th submesh uniformly if possible
15842 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
15843 {
15844 unsigned nref = nrefine_for_mesh[imesh];
15845 for (unsigned i = 0; i < nref; i++)
15846 {
15847 mmesh_pt->p_refine_uniformly(doc_info);
15848 }
15849 }
15850 else
15851 {
15852 oomph_info << "Info/Warning: Cannot p-refine mesh " << imesh
15853 << std::endl;
15854 }
15855 }
15856 // Rebuild the global mesh
15858 }
15859
15861 {
15863 oomph_info << "Time for mesh-level mesh refinement in "
15864 << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15865 << std::endl;
15867 }
15868
15869 // Any actions after the adaptation phase
15871
15872
15874 {
15877 << "Time for actions after adapt Problem::p_refine_uniformly_aux(): "
15878 << t_end - t_start << std::endl;
15880 }
15881
15882
15883#ifdef OOMPH_HAS_MPI
15884
15885 // Prune it?
15886 if (prune)
15887 {
15888 // Note: This calls assign eqn numbers already...
15892
15894 {
15896 oomph_info << "Time for Problem::prune_halo_elements_and_nodes() in "
15897 << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15898 << std::endl;
15899 }
15900 }
15901 else
15902#else
15903 if (prune)
15904 {
15905 std::ostringstream error_message;
15906 error_message
15907 << "Requested pruning in serial build. Ignoring the request.\n";
15908 OomphLibWarning(error_message.str(),
15909 "Problem::p_refine_uniformly_aux()",
15911 }
15912#endif
15913 {
15914 // Do equation numbering
15916 << "Number of equations after Problem::p_refine_uniformly_aux(): "
15917 << assign_eqn_numbers() << std::endl;
15918
15920 {
15922 oomph_info << "Time for Problem::assign_eqn_numbers() in "
15923 << "Problem::p_refine_uniformly_aux(): " << t_end - t_start
15924 << std::endl;
15925 }
15926 }
15927 }
15928
15929 //========================================================================
15930 /// Refine submesh i_mesh uniformly and rebuild problem;
15931 /// doc refinement process.
15932 //========================================================================
15933 void Problem::refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15934 {
15936
15937#ifdef PARANOID
15938 // Number of submeshes?
15939 if (i_mesh >= nsub_mesh())
15940 {
15941 std::ostringstream error_message;
15942 error_message << "imesh " << i_mesh
15943 << " is greater than the number of sub meshes "
15944 << nsub_mesh() << std::endl;
15945
15946 throw OomphLibError(
15947 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15948 }
15949#endif
15950
15951 // Refine single mesh uniformly if possible
15953 dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15954 {
15955 mmesh_pt->refine_uniformly(doc_info);
15956 }
15957 else
15958 {
15959 oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
15960 << std::endl;
15961 }
15962
15963 // Rebuild the global mesh
15965
15966 // Any actions after the adaptation phase
15968
15969 // Do equation numbering
15970 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
15971 }
15972
15973 //========================================================================
15974 /// p-refine submesh i_mesh uniformly and rebuild problem;
15975 /// doc refinement process.
15976 //========================================================================
15977 void Problem::p_refine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
15978 {
15980
15981#ifdef PARANOID
15982 // Number of submeshes?
15983 if (i_mesh >= nsub_mesh())
15984 {
15985 std::ostringstream error_message;
15986 error_message << "imesh " << i_mesh
15987 << " is greater than the number of sub meshes "
15988 << nsub_mesh() << std::endl;
15989
15990 throw OomphLibError(
15991 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
15992 }
15993#endif
15994
15995 // Refine single mesh uniformly if possible
15997 dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
15998 {
15999 mmesh_pt->p_refine_uniformly(doc_info);
16000 }
16001 else
16002 {
16003 oomph_info << "Info/Warning: Mesh cannot be refined uniformly "
16004 << std::endl;
16005 }
16006
16007 // Rebuild the global mesh
16009
16010 // Any actions after the adaptation phase
16012
16013 // Do equation numbering
16014 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16015 }
16016
16017
16018 //========================================================================
16019 /// Unrefine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
16020 /// Return 0 for success,
16021 /// 1 for failure (if unrefinement has reached the coarsest permitted
16022 /// level)
16023 //========================================================================
16025 {
16026 // Call actions_before_adapt()
16028
16029 // Has unrefinement been successful?
16030 unsigned success_flag = 0;
16031
16032 // Number of submeshes?
16033 unsigned n_mesh = nsub_mesh();
16034
16035 // Single mesh:
16036 if (n_mesh == 0)
16037 {
16038 // Unrefine single mesh uniformly if possible
16040 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
16041 {
16042 success_flag += mmesh_pt->unrefine_uniformly();
16043 }
16044 else
16045 {
16046 oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
16047 << std::endl;
16048 }
16049 }
16050 // Multiple submeshes
16051 else
16052 {
16053 // Loop over submeshes
16054 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
16055 {
16056 // Unrefine i-th submesh uniformly if possible
16058 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
16059 {
16060 success_flag += mmesh_pt->unrefine_uniformly();
16061 }
16062 else
16063 {
16064 oomph_info << "Info/Warning: Cannot unrefine mesh " << imesh
16065 << std::endl;
16066 }
16067 }
16068 // Rebuild the global mesh
16070 }
16071
16072 // Any actions after the adaptation phase
16074
16075 // Do equation numbering
16076 oomph_info << " Number of equations: " << assign_eqn_numbers() << std::endl;
16077
16078 // Judge success
16079 if (success_flag > 0)
16080 {
16081 return 1;
16082 }
16083 else
16084 {
16085 return 0;
16086 }
16087 }
16088
16089 //========================================================================
16090 /// Unrefine submesh i_mesh uniformly and rebuild problem.
16091 /// Return 0 for success,
16092 /// 1 for failure (if unrefinement has reached the coarsest permitted
16093 /// level)
16094 //========================================================================
16095 unsigned Problem::unrefine_uniformly(const unsigned& i_mesh)
16096 {
16098
16099 // Has unrefinement been successful?
16100 unsigned success_flag = 0;
16101
16102#ifdef PARANOID
16103 // Number of submeshes?
16104 if (i_mesh >= nsub_mesh())
16105 {
16106 std::ostringstream error_message;
16107 error_message << "imesh " << i_mesh
16108 << " is greater than the number of sub meshes "
16109 << nsub_mesh() << std::endl;
16110
16111 throw OomphLibError(
16112 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
16113 }
16114#endif
16115
16116 // Unrefine single mesh uniformly if possible
16118 dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
16119 {
16120 success_flag += mmesh_pt->unrefine_uniformly();
16121 }
16122 else
16123 {
16124 oomph_info << "Info/Warning: Mesh cannot be unrefined uniformly "
16125 << std::endl;
16126 }
16127
16128 // Rebuild the global mesh
16130
16131 // Any actions after the adaptation phase
16133
16134 // Do equation numbering
16135 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16136
16137 // Judge success
16138 if (success_flag > 0)
16139 {
16140 return 1;
16141 }
16142 else
16143 {
16144 return 0;
16145 }
16146 }
16147
16148
16149 //========================================================================
16150 /// p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem;
16151 /// doc refinement process.
16152 //========================================================================
16154 {
16156
16157 // Number of submeshes?
16158 unsigned n_mesh = nsub_mesh();
16159
16160 // Single mesh:
16161 if (n_mesh == 0)
16162 {
16163 // Unrefine single mesh uniformly if possible
16165 dynamic_cast<RefineableMeshBase*>(mesh_pt(0)))
16166 {
16167 mmesh_pt->p_unrefine_uniformly(doc_info);
16168 }
16169 else
16170 {
16171 oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16172 << std::endl;
16173 }
16174 }
16175 // Multiple submeshes
16176 else
16177 {
16178 // Not tested:
16179 throw OomphLibError("This functionality has not yet been tested.",
16182 // Loop over submeshes
16183 for (unsigned imesh = 0; imesh < n_mesh; imesh++)
16184 {
16185 // Unrefine i-th submesh uniformly if possible
16187 dynamic_cast<RefineableMeshBase*>(mesh_pt(imesh)))
16188 {
16189 mmesh_pt->p_unrefine_uniformly(doc_info);
16190 }
16191 else
16192 {
16193 oomph_info << "Info/Warning: Cannot p-unrefine mesh " << imesh
16194 << std::endl;
16195 }
16196 }
16197 // Rebuild the global mesh
16199 }
16200
16201 // Any actions after the adaptation phase
16203
16204 // Do equation numbering
16205 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16206 }
16207
16208 //========================================================================
16209 /// p-unrefine submesh i_mesh uniformly and rebuild problem;
16210 /// doc refinement process.
16211 //========================================================================
16212 void Problem::p_unrefine_uniformly(const unsigned& i_mesh, DocInfo& doc_info)
16213 {
16215
16216#ifdef PARANOID
16217 // Number of submeshes?
16218 if (i_mesh >= nsub_mesh())
16219 {
16220 std::ostringstream error_message;
16221 error_message << "imesh " << i_mesh
16222 << " is greater than the number of sub meshes "
16223 << nsub_mesh() << std::endl;
16224
16225 throw OomphLibError(
16226 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
16227 }
16228#endif
16229
16230 // Refine single mesh uniformly if possible
16232 dynamic_cast<RefineableMeshBase*>(mesh_pt(i_mesh)))
16233 {
16234 mmesh_pt->p_unrefine_uniformly(doc_info);
16235 }
16236 else
16237 {
16238 oomph_info << "Info/Warning: Mesh cannot be p-unrefined uniformly "
16239 << std::endl;
16240 }
16241
16242 // Rebuild the global mesh
16244
16245 // Any actions after the adaptation phase
16247
16248 // Do equation numbering
16249 oomph_info << "Number of equations: " << assign_eqn_numbers() << std::endl;
16250 }
16251
16252
16253 //========================================================================
16254 /// Do one timestep, dt, forward using Newton's method with specified
16255 /// tolerance and linear solver specified via member data.
16256 /// Keep adapting on all meshes to criteria specified in
16257 /// these meshes (up to max_adapt adaptations are performed).
16258 /// If first_timestep==true, re-set initial conditions after mesh adaptation.
16259 /// Shifting of time can be suppressed by overwriting the
16260 /// default value of shift (true). [Shifting must be done
16261 /// if first_timestep==true because we're constantly re-assigning
16262 /// the initial conditions; if first_timestep==true and shift==false
16263 /// shifting is performed anyway and a warning is issued.
16264 //========================================================================
16265 void Problem::unsteady_newton_solve(const double& dt,
16266 const unsigned& max_adapt,
16267 const bool& first_timestep,
16268 const bool& shift)
16269 {
16270 // Do shifting or not?
16271 bool shift_it = shift;
16272
16273 // Warning:
16275 {
16276 shift_it = true;
16278 << "\n\n===========================================================\n";
16279 oomph_info << " ******** WARNING *********** \n";
16281 << "===========================================================\n";
16282 oomph_info << "Problem::unsteady_newton_solve() called with "
16283 << std::endl;
16284 oomph_info << "first_timestep: " << first_timestep << std::endl;
16285 oomph_info << "shift: " << shift << std::endl;
16286 oomph_info << "This doesn't make sense (shifting does have to be done"
16287 << std::endl;
16288 oomph_info << "since we're constantly re-assigning the initial conditions"
16289 << std::endl;
16291 << "\n===========================================================\n\n";
16292 }
16293
16294
16295 // Find the initial time
16296 double initial_time = time_pt()->time();
16297
16298 // Max number of solves
16299 unsigned max_solve = max_adapt + 1;
16300
16301 // Adaptation loop
16302 //----------------
16303 for (unsigned isolve = 0; isolve < max_solve; isolve++)
16304 {
16305 // Only adapt after the first solve has been done!
16306 if (isolve > 0)
16307 {
16308 unsigned n_refined;
16309 unsigned n_unrefined;
16310
16311 // Adapt problem
16313
16314#ifdef OOMPH_HAS_MPI
16315 // Adaptation only converges if ALL the processes have no
16316 // refinement or unrefinement to perform
16317 unsigned total_refined = 0;
16318 unsigned total_unrefined = 0;
16320 {
16323 1,
16325 MPI_SUM,
16326 this->communicator_pt()->mpi_comm());
16330 1,
16332 MPI_SUM,
16333 this->communicator_pt()->mpi_comm());
16335 }
16336#endif
16337
16338 oomph_info << "---> " << n_refined << " elements were refined, and "
16339 << n_unrefined << " were unrefined, in total." << std::endl;
16340
16341 // Check convergence of adaptation cycle
16342 if ((n_refined == 0) && (n_unrefined == 0))
16343 {
16344 oomph_info << "\n \n Solution is fully converged in "
16345 << "Problem::unsteady_newton_solver() \n \n ";
16346 break;
16347 }
16348
16349 // Reset the time
16350 time_pt()->time() = initial_time;
16351
16352 // Reset the inital condition on refined meshes. Note that because we
16353 // have reset the global time to the initial time, the initial
16354 // conditions are reset at time t=0 rather than at time t=dt
16355 if (first_timestep)
16356 {
16357 // Reset default set_initial_condition has been called flag to false
16359
16360 oomph_info << "Re-setting initial condition " << std::endl;
16362
16363 // If the default set_initial_condition function has been called,
16364 // we must not shift the timevalues on the first timestep, as we
16365 // will NOT be constantly re-assigning the initial condition
16367 {
16368 shift_it = false;
16369 }
16370 }
16371 }
16372
16373 // Now do the actual unsteady timestep
16374 // If it's the first time around the loop, or the first timestep
16375 // shift the timevalues, otherwise don't
16376 // Note: we need to shift if it's the first timestep because
16377 // we're constantly re-assigning the initial condition above!
16378 // The only exception to this is if the default set_initial_condition
16379 // function has been called, in which case we must NOT shift!
16380 if ((isolve == 0) || (first_timestep))
16381 {
16383 }
16384 // Subsequent solve: Have shifted already -- don't do it again.
16385 else
16386 {
16387 shift_it = false;
16389 }
16390
16391 if (isolve == max_solve - 1)
16392 {
16394 << std::endl
16395 << "----------------------------------------------------------"
16396 << std::endl
16397 << "Reached max. number of adaptations in \n"
16398 << "Problem::unsteady_newton_solver().\n"
16399 << "----------------------------------------------------------"
16400 << std::endl
16401 << std::endl;
16402 }
16403
16404 } // End of adaptation loop
16405 }
16406
16407
16408 //========================================================================
16409 /// Adaptive Newton solver.
16410 /// The linear solver takes a pointer to the problem (which defines
16411 /// the Jacobian \b J and the residual Vector \b r) and returns
16412 /// the solution \b x of the system
16413 /// \f[ {\bf J} {\bf x} = - \bf{r} \f].
16414 /// Performs at most max_adapt adaptations on all meshes.
16415 //========================================================================
16416 void Problem::newton_solve(const unsigned& max_adapt)
16417 {
16418 // Max number of solves
16419 unsigned max_solve = max_adapt + 1;
16420
16421 // Adaptation loop
16422 //----------------
16423 for (unsigned isolve = 0; isolve < max_solve; isolve++)
16424 {
16425 // Only adapt after the first solve has been done!
16426 if (isolve > 0)
16427 {
16428 unsigned n_refined;
16429 unsigned n_unrefined;
16430
16431 // Adapt problem
16433
16434#ifdef OOMPH_HAS_MPI
16435 // Adaptation only converges if ALL the processes have no
16436 // refinement or unrefinement to perform
16437 unsigned total_refined = 0;
16438 unsigned total_unrefined = 0;
16440 {
16443 1,
16445 MPI_SUM,
16446 this->communicator_pt()->mpi_comm());
16450 1,
16452 MPI_SUM,
16453 this->communicator_pt()->mpi_comm());
16455 }
16456#endif
16457
16458 oomph_info << "---> " << n_refined << " elements were refined, and "
16459 << n_unrefined << " were unrefined"
16460#ifdef OOMPH_HAS_MPI
16461 << ", in total (over all processors).\n";
16462#else
16463 << ".\n";
16464#endif
16465
16466
16467 // Check convergence of adaptation cycle
16468 if ((n_refined == 0) && (n_unrefined == 0))
16469 {
16470 oomph_info << "\n \n Solution is fully converged in "
16471 << "Problem::newton_solver(). \n \n ";
16472 break;
16473 }
16474 }
16475
16476
16477 // Do actual solve
16478 //----------------
16479 {
16480 // Now update anything that needs updating
16481 // NOT NEEDED -- IS CALLED IN newton_solve BELOW! #
16482 // actions_before_newton_solve();
16483
16484 try
16485 {
16486 // Solve the non-linear problem for this timestep with Newton's method
16487 newton_solve();
16488 }
16489 // Catch any exceptions thrown in the Newton solver
16490 catch (NewtonSolverError& error)
16491 {
16492 oomph_info << std::endl
16493 << "USER-DEFINED ERROR IN NEWTON SOLVER " << std::endl;
16494 // Check to see whether we have reached Max_iterations
16495 if (error.iterations == Max_newton_iterations)
16496 {
16497 oomph_info << "MAXIMUM NUMBER OF ITERATIONS (" << error.iterations
16498 << ") REACHED WITHOUT CONVERGENCE " << std::endl;
16499 }
16500 // If not, it must be that we have exceeded the maximum residuals
16501 else
16502 {
16503 oomph_info << "MAXIMUM RESIDUALS: " << error.maxres
16504 << "EXCEEDS PREDEFINED MAXIMUM " << Max_residuals
16505 << std::endl;
16506 }
16507
16508 // Die horribly!!
16509 std::ostringstream error_stream;
16510 error_stream << "Error occured in adaptive Newton solver. "
16511 << std::endl;
16512 throw OomphLibError(error_stream.str(),
16515 }
16516
16517 // Now update anything that needs updating
16518 // NOT NEEDED -- WAS CALLED IN newton_solve ABOVE
16519 // !actions_after_newton_solve();
16520
16521 } // End of solve block
16522
16523
16524 if (isolve == max_solve - 1)
16525 {
16527 << std::endl
16528 << "----------------------------------------------------------"
16529 << std::endl
16530 << "Reached max. number of adaptations in \n"
16531 << "Problem::newton_solver().\n"
16532 << "----------------------------------------------------------"
16533 << std::endl
16534 << std::endl;
16535 }
16536
16537 } // End of adaptation loop
16538 }
16539
16540 //========================================================================
16541 /// Delete any external storage for any submeshes
16542 /// NB this would ordinarily take place within the adaptation procedure
16543 /// for each submesh (See RefineableMesh::adapt_mesh(...)), but there
16544 /// are instances where the actions_before/after_adapt routines are used
16545 /// and no adaptive routines are called in between (e.g. when doc-ing
16546 /// errors at the end of an adaptive newton solver)
16547 //========================================================================
16549 {
16550 // Number of submeshes
16551 unsigned n_mesh = nsub_mesh();
16552
16553 // External storage will only exist if there is more than one (sub)mesh
16554 if (n_mesh > 1)
16555 {
16556 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16557 {
16559 }
16560 }
16561 }
16562
16563
16564#ifdef OOMPH_HAS_MPI
16565
16566 //====================================================================
16567 /// Get all the halo data stored on this processor and store pointers
16568 /// to the data in a map, indexed by the gobal eqn number
16569 //====================================================================
16570 void Problem::get_all_halo_data(std::map<unsigned, double*>& map_of_halo_data)
16571 {
16572 // Halo data is stored in the meshes, so kick the problem down to that
16573 // level
16574
16575 // Find the number of meshes
16576 unsigned n_mesh = this->nsub_mesh();
16577 // If there are no submeshes it's only the main mesh
16578 if (n_mesh == 0)
16579 {
16581 }
16582 // Otherwise loop over all the submeshes
16583 else
16584 {
16585 for (unsigned imesh = 0; imesh < n_mesh; ++imesh)
16586 {
16588 }
16589 }
16590 }
16591
16592
16593 //========================================================================
16594 /// Check the halo/haloed/shared node/element schemes.
16595 //========================================================================
16597 {
16598 // The bulk of the stuff that was in this routine is mesh-based, and
16599 // should therefore drop into the Mesh base class. All that needs to remain
16600 // here is a "wrapper" which calls the function dependent upon the number
16601 // of (sub)meshes that may have been distributed.
16602
16603 unsigned n_mesh = nsub_mesh();
16604
16605 if (n_mesh == 0)
16606 {
16607 oomph_info << "Checking halo schemes on single mesh" << std::endl;
16608 doc_info.label() = "_one_and_only_mesh_";
16609 mesh_pt()->check_halo_schemes(doc_info,
16611 }
16612 else // there are submeshes
16613 {
16614 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
16615 {
16616 oomph_info << "Checking halo schemes on submesh " << i_mesh
16617 << std::endl;
16618 std::stringstream tmp;
16619 tmp << "_mesh" << i_mesh << "_";
16620 doc_info.label() = tmp.str();
16623 }
16624 }
16625 }
16626
16627
16628 //========================================================================
16629 /// Synchronise all dofs by calling the appropriate synchronisation
16630 /// routines for all meshes and the assembly handler
16631 //========================================================================
16633 {
16634 // Synchronise dofs themselves
16635 bool do_halos = true;
16636 bool do_external_halos = false;
16637 this->synchronise_dofs(do_halos, do_external_halos);
16638
16639
16640 do_halos = false;
16641 do_external_halos = true;
16642 this->synchronise_dofs(do_halos, do_external_halos);
16643
16644 // Now perform any synchronisation required by the assembly handler
16646 }
16647
16648
16649 //========================================================================
16650 /// Synchronise the degrees of freedom by overwriting
16651 /// the haloed values with their non-halo counterparts held
16652 /// on other processors. Bools control if we deal with data associated with
16653 /// external halo/ed elements/nodes or the "normal" halo/ed ones.
16654 //========================================================================
16656 const bool& do_external_halos)
16657 {
16658 // Do we have submeshes?
16659 unsigned n_mesh_loop = 1;
16660 unsigned nmesh = nsub_mesh();
16661 if (nmesh > 0)
16662 {
16663 n_mesh_loop = nmesh;
16664 }
16665
16666 // Local storage for number of processors and current processor
16667 const int n_proc = this->communicator_pt()->nproc();
16668
16669 // If only one processor then return
16670 if (n_proc == 1)
16671 {
16672 return;
16673 }
16674
16675 const int my_rank = this->communicator_pt()->my_rank();
16676
16677 // Storage for number of data to be sent to each processor
16679
16680 // Storage for all values to be sent to all processors
16682
16683 // Start location within send_data for data to be sent to each processor
16685
16686 // Loop over all processors
16687 for (int rank = 0; rank < n_proc; rank++)
16688 {
16689 // Set the offset for the current processor
16691
16692 // Don't bother to do anything if the processor in the loop is the
16693 // current processor
16694 if (rank != my_rank)
16695 {
16696 // Deal with sub-meshes one-by-one if required
16697 Mesh* my_mesh_pt = 0;
16698
16699 // Loop over submeshes
16700 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16701 {
16702 if (nmesh == 0)
16703 {
16704 my_mesh_pt = mesh_pt();
16705 }
16706 else
16707 {
16709 }
16710
16711 if (do_halos)
16712 {
16713 // How many of my nodes are haloed by the processor whose values
16714 // are updated?
16715 unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
16716 for (unsigned n = 0; n < n_nod; n++)
16717 {
16718 // Add the data for each haloed node to the vector
16719 my_mesh_pt->haloed_node_pt(rank, n)->add_values_to_vector(
16720 send_data);
16721 }
16722
16723 // Now loop over haloed elements and prepare to add their
16724 // internal data to the big vector to be sent
16726 my_mesh_pt->haloed_element_pt(rank);
16727 unsigned nelem_haloed = haloed_elem_pt.size();
16728 for (unsigned e = 0; e < nelem_haloed; e++)
16729 {
16731 }
16732 }
16733
16735 {
16736 // How many of my nodes are externally haloed by the processor whose
16737 // values are updated? NB these nodes are on the external mesh.
16738 unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
16739 for (unsigned n = 0; n < n_ext_nod; n++)
16740 {
16741 // Add data from each external haloed node to the vector
16742 my_mesh_pt->external_haloed_node_pt(rank, n)
16743 ->add_values_to_vector(send_data);
16744 }
16745
16746 // Now loop over haloed elements and prepare to send internal data
16747 unsigned next_elem_haloed =
16748 my_mesh_pt->nexternal_haloed_element(rank);
16749 for (unsigned e = 0; e < next_elem_haloed; e++)
16750 {
16751 my_mesh_pt->external_haloed_element_pt(rank, e)
16753 }
16754 }
16755 } // end of loop over meshes
16756 }
16757
16758 // Find the number of data added to the vector
16760 }
16761
16762
16763 // Storage for the number of data to be received from each processor
16765
16766 // Now send numbers of data to be sent between all processors
16767 MPI_Alltoall(&send_n[0],
16768 1,
16769 MPI_INT,
16770 &receive_n[0],
16771 1,
16772 MPI_INT,
16773 this->communicator_pt()->mpi_comm());
16774
16775 // We now prepare the data to be received
16776 // by working out the displacements from the received data
16778 int receive_data_count = 0;
16779 for (int rank = 0; rank < n_proc; ++rank)
16780 {
16781 // Displacement is number of data received so far
16784 }
16785
16786 // Now resize the receive buffer for all data from all processors
16787 // Make sure that it has a size of at least one
16788 if (receive_data_count == 0)
16789 {
16791 }
16793
16794 // Make sure that the send buffer has size at least one
16795 // so that we don't get a segmentation fault
16796 if (send_data.size() == 0)
16797 {
16798 send_data.resize(1);
16799 }
16800
16801 // Now send the data between all the processors
16803 &send_n[0],
16805 MPI_DOUBLE,
16806 &receive_data[0],
16807 &receive_n[0],
16809 MPI_DOUBLE,
16810 this->communicator_pt()->mpi_comm());
16811
16812 // Now use the received data to update the halo nodes
16813 for (int send_rank = 0; send_rank < n_proc; send_rank++)
16814 {
16815 // Don't bother to do anything for the processor corresponding to the
16816 // current processor or if no data were received from this processor
16817 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
16818 {
16819 // Counter for the data within the large array
16821
16822 // Deal with sub-meshes one-by-one if required
16823 Mesh* my_mesh_pt = 0;
16824
16825 // Loop over submeshes
16826 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
16827 {
16828 if (nmesh == 0)
16829 {
16830 my_mesh_pt = mesh_pt();
16831 }
16832 else
16833 {
16835 }
16836
16837 if (do_halos)
16838 {
16839 // How many of my nodes are halos whose non-halo counter
16840 // parts live on processor send_rank?
16841 unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
16842 for (unsigned n = 0; n < n_nod; n++)
16843 {
16844 // Read in values for each halo node
16845 my_mesh_pt->halo_node_pt(send_rank, n)
16846 ->read_values_from_vector(receive_data, count);
16847 }
16848
16849 // Get number of halo elements whose non-halo is
16850 // on process send_rank
16852 my_mesh_pt->halo_element_pt(send_rank);
16853
16854 unsigned nelem_halo = halo_elem_pt.size();
16855 for (unsigned e = 0; e < nelem_halo; e++)
16856 {
16859 }
16860 }
16861
16863 {
16864 // How many of my nodes are external halos whose external non-halo
16865 // counterparts live on processor send_rank?
16866 unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
16867
16868 // Copy into the values of the external halo nodes
16869 // on the present processors
16870 for (unsigned n = 0; n < n_ext_nod; n++)
16871 {
16872 // Read the data from the array into each halo node
16873 my_mesh_pt->external_halo_node_pt(send_rank, n)
16874 ->read_values_from_vector(receive_data, count);
16875 }
16876
16877 // Get number of halo elements whose non-halo is
16878 // on process send_rank
16879 unsigned next_elem_halo =
16880 my_mesh_pt->nexternal_halo_element(send_rank);
16881 for (unsigned e = 0; e < next_elem_halo; e++)
16882 {
16883 my_mesh_pt->external_halo_element_pt(send_rank, e)
16885 }
16886 }
16887
16888 } // end of loop over meshes
16889 }
16890 } // End of data is received
16891 } // End of synchronise
16892
16893
16894 //========================================================================
16895 /// Synchronise equation numbers and return the total
16896 /// number of degrees of freedom in the overall problem
16897 //========================================================================
16898 long Problem::synchronise_eqn_numbers(const bool& assign_local_eqn_numbers)
16899 {
16900 // number of equations on this processor, which at this stage is only known
16901 // by counting the number of dofs that have been added to the problem
16902 unsigned my_n_eqn = Dof_pt.size();
16903
16904 // my rank
16905 unsigned my_rank = Communicator_pt->my_rank();
16906
16907 // number of processors
16908 unsigned nproc = Communicator_pt->nproc();
16909
16910 // // Time alternative communication
16911 // Vector<unsigned> n_eqn(nproc);
16912 // {
16913 // double t_start = TimingHelpers::timer();
16914
16915 // // Gather numbers of equations (enumerated independently on all procs)
16916 // MPI_Allgather(&my_n_eqn,1,MPI_UNSIGNED,&n_eqn[0],
16917 // 1,MPI_INT,Communicator_pt->mpi_comm());
16918
16919 // double t_end = TimingHelpers::timer();
16920 // oomph_info << "Time for allgather-based exchange of eqn numbers: "
16921 // << t_end-t_start << std::endl;
16922 // }
16923
16924 double t_start = TimingHelpers::timer();
16925
16926 // send my_n_eqn to with rank greater than my_rank
16927 unsigned n_send = nproc - my_rank - 1;
16929 for (unsigned p = my_rank + 1; p < nproc; p++)
16930 {
16932 1,
16934 p,
16935 0,
16936 Communicator_pt->mpi_comm(),
16937 &send_req[p - my_rank - 1]);
16938 }
16939
16940 // recv n_eqn from processors with rank less than my_rank
16942 for (unsigned p = 0; p < my_rank; p++)
16943 {
16945 1,
16947 p,
16948 0,
16949 Communicator_pt->mpi_comm(),
16951 }
16952
16953 double t_end = 0.0;
16955 {
16957 oomph_info << "Time for send and receive stuff: " << t_end - t_start
16958 << std::endl;
16960 }
16961
16962 // determine the number of equation on processors with rank
16963 // less than my_rank
16964 unsigned my_eqn_num_base = 0;
16965 for (unsigned p = 0; p < my_rank; p++)
16966 {
16968 // if (n_eqn_on_proc[p]!=n_eqn[p])
16969 // {
16970 // std::cout << "proc " << my_rank << "clash in eqn numbers: "
16971 // << p << " " << n_eqn_on_proc[p] << " " << n_eqn[p]
16972 // << std::endl;
16973 // }
16974 }
16975
16976 // Loop over all internal data (on elements) and bump up their
16977 // equation numbers if they exist
16978 unsigned nelem = mesh_pt()->nelement();
16979 for (unsigned e = 0; e < nelem; e++)
16980 {
16982
16983 unsigned nintern_data = el_pt->ninternal_data();
16984 for (unsigned iintern = 0; iintern < nintern_data; iintern++)
16985 {
16987 unsigned nval = int_data_pt->nvalue();
16988 for (unsigned ival = 0; ival < nval; ival++)
16989 {
16991 if (old_eqn_number >= 0) // i.e. it's being used
16992 {
16993 // Bump up eqn number
16996 }
16997 }
16998 }
16999 }
17000
17001 // Loop over all nodes on current processor and bump up their
17002 // equation numbers if they're not pinned!
17003 unsigned nnod = mesh_pt()->nnode();
17004 for (unsigned j = 0; j < nnod; j++)
17005 {
17006 Node* nod_pt = mesh_pt()->node_pt(j);
17007
17008 // loop over ALL eqn numbers - variable number of values
17009 unsigned nval = nod_pt->nvalue();
17010
17011 for (unsigned ival = 0; ival < nval; ival++)
17012 {
17014 // Include all eqn numbers
17015 if (old_eqn_number >= 0)
17016 {
17017 // Bump up eqn number
17020 }
17021 }
17022
17023 // Is this a solid node? If so, need to bump up its equation number(s)
17024 SolidNode* solid_nod_pt = dynamic_cast<SolidNode*>(nod_pt);
17025
17026 if (solid_nod_pt != 0)
17027 {
17028 // Find equation numbers
17029 unsigned nval = solid_nod_pt->variable_position_pt()->nvalue();
17030 for (unsigned ival = 0; ival < nval; ival++)
17031 {
17032 int old_eqn_number =
17033 solid_nod_pt->variable_position_pt()->eqn_number(ival);
17034 // include all eqn numbers
17035
17036 if (old_eqn_number >= 0)
17037 {
17038 // Bump up eqn number
17040 solid_nod_pt->variable_position_pt()->eqn_number(ival) =
17042 }
17043 }
17044 }
17045 }
17046
17048 {
17050 oomph_info << "Time for bumping: " << t_end - t_start << std::endl;
17052 }
17053
17054
17055 // Now copy the haloed eqn numbers across
17056 // This has to include the internal data equation numbers as well
17057 // as the solid node equation numbers
17058 bool do_halos = true;
17059 bool do_external_halos = false;
17061
17063 {
17065 oomph_info << "Time for copy_haloed_eqn_numbers_helper for halos: "
17066 << t_end - t_start << std::endl;
17068 }
17069
17070 // Now do external halo stuff
17071 do_halos = false;
17072 do_external_halos = true;
17074
17076 {
17079 << "Time for copy_haloed_eqn_numbers_helper for external halos: "
17080 << t_end - t_start << std::endl;
17082 }
17083
17084 // Now the global equation numbers have been updated.
17085 //---------------------------------------------------
17086 // Setup the local equation numbers again.
17087 //----------------------------------------
17088 if (assign_local_eqn_numbers)
17089 {
17090 // Loop over the submeshes: Note we need to call the submeshes' own
17091 // assign_*_eqn_number() otherwise we miss additional functionality
17092 // that is implemented (e.g.) in SolidMeshes!
17093 unsigned n_sub_mesh = nsub_mesh();
17094 if (n_sub_mesh == 0)
17095 {
17097 }
17098 else
17099 {
17100 for (unsigned i = 0; i < n_sub_mesh; i++)
17101 {
17103 }
17104 }
17105 }
17106
17108 {
17110 oomph_info << "Time for assign_local_eqn_numbers in sync: "
17111 << t_end - t_start << std::endl;
17113 }
17114
17115 // wait for the sends to complete
17116 if (n_send > 0)
17117 {
17120 }
17121
17123 {
17125 oomph_info << "Time for waitall: " << t_end - t_start << std::endl;
17127 }
17128
17129 // build the Dof distribution pt
17131
17132 // and return the total number of equations in the problem
17133 return (long)Dof_distribution_pt->nrow();
17134 }
17135
17136
17137 //=======================================================================
17138 /// A private helper function to
17139 /// copy the haloed equation numbers into the halo equation numbers,
17140 /// either for the problem's one and only mesh or for all of its
17141 /// submeshes. Bools control if we deal with data associated with
17142 /// external halo/ed elements/nodes or the "normal" halo/ed ones.
17143 //===================================================================
17145 const bool& do_external_halos)
17146 {
17147 // Do we have submeshes?
17148 unsigned n_mesh_loop = 1;
17149 unsigned nmesh = nsub_mesh();
17150 if (nmesh > 0)
17151 {
17152 n_mesh_loop = nmesh;
17153 }
17154
17155 // Storage for number of processors and current processor
17156 int n_proc = this->communicator_pt()->nproc();
17157
17158 // If only one processor then return
17159 if (n_proc == 1)
17160 {
17161 return;
17162 }
17163 int my_rank = this->communicator_pt()->my_rank();
17164
17165 // Storage for number of data to be sent to each processor
17167 // Storage for all equation numbers to be sent to all processors
17169 // Start location within send_data for data to be sent to each processor
17171
17172
17173 // Loop over all processors whose eqn numbers are to be updated
17174 for (int rank = 0; rank < n_proc; rank++)
17175 {
17176 // Set the displacement of the current processor in the loop
17178
17179 // If I'm not the processor whose halo eqn numbers are updated,
17180 // some of my nodes may be haloed: Stick their
17181 // eqn numbers into the vector
17182 if (rank != my_rank)
17183 {
17184 // Deal with sub-meshes one-by-one if required
17185 Mesh* my_mesh_pt = 0;
17186
17187 // Loop over submeshes
17188 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17189 {
17190 if (nmesh == 0)
17191 {
17192 my_mesh_pt = mesh_pt();
17193 }
17194 else
17195 {
17197 }
17198
17199 if (do_halos)
17200 {
17201 // Add equation numbers for each haloed node
17202 unsigned n_nod = my_mesh_pt->nhaloed_node(rank);
17203 for (unsigned n = 0; n < n_nod; n++)
17204 {
17205 my_mesh_pt->haloed_node_pt(rank, n)->add_eqn_numbers_to_vector(
17206 send_data);
17207 }
17208
17209 // Add the equation numbers associated with internal data
17210 // in the haloed elements
17212 my_mesh_pt->haloed_element_pt(rank);
17213 unsigned nelem_haloed = haloed_elem_pt.size();
17214 for (unsigned e = 0; e < nelem_haloed; e++)
17215 {
17217 }
17218 }
17219
17221 {
17222 // Add equation numbers associated with external haloed nodes
17223 unsigned n_ext_nod = my_mesh_pt->nexternal_haloed_node(rank);
17224 for (unsigned n = 0; n < n_ext_nod; n++)
17225 {
17226 my_mesh_pt->external_haloed_node_pt(rank, n)
17227 ->add_eqn_numbers_to_vector(send_data);
17228 }
17229
17230 // Add the equation numbers associated with internal data in
17231 // each external haloed element
17232 unsigned next_elem_haloed =
17233 my_mesh_pt->nexternal_haloed_element(rank);
17234 for (unsigned e = 0; e < next_elem_haloed; e++)
17235 {
17236 // how many internal data values for this element?
17237 my_mesh_pt->external_haloed_element_pt(rank, e)
17239 }
17240 }
17241
17242 } // end of loop over meshes
17243 }
17244
17245 // Find the number of data added to the vector by this processor
17247 }
17248
17249 // Storage for the number of data to be received from each processor
17251
17252 // Communicate all numbers of data to be sent between all processors
17253 MPI_Alltoall(&send_n[0],
17254 1,
17255 MPI_INT,
17256 &receive_n[0],
17257 1,
17258 MPI_INT,
17259 this->communicator_pt()->mpi_comm());
17260
17261 // We now prepare the data to be received
17262 // by working out the displacements from the received data
17264 int receive_data_count = 0;
17265 for (int rank = 0; rank < n_proc; ++rank)
17266 {
17267 // Displacement is number of data received so far
17270 }
17271
17272 // Now resize the receive buffer
17273 // Make sure that it has a size of at least one
17274 if (receive_data_count == 0)
17275 {
17277 }
17279
17280 // Make sure that the send buffer has size at least one
17281 // so that we don't get a segmentation fault
17282 if (send_data.size() == 0)
17283 {
17284 send_data.resize(1);
17285 }
17286
17287 // Now send the data between all the processors
17289 &send_n[0],
17291 MPI_LONG,
17292 &receive_data[0],
17293 &receive_n[0],
17295 MPI_LONG,
17296 this->communicator_pt()->mpi_comm());
17297
17298
17299 // Loop over all other processors to receive their
17300 // eqn numbers
17301 for (int send_rank = 0; send_rank < n_proc; send_rank++)
17302 {
17303 // Don't do anything for the processor corresponding to the
17304 // current processor or if no data were received from this processor
17305 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
17306 {
17307 // Counter for the data within the large array
17309
17310 // Deal with sub-meshes one-by-one if required
17311 Mesh* my_mesh_pt = 0;
17312
17313 // Loop over submeshes
17314 for (unsigned imesh = 0; imesh < n_mesh_loop; imesh++)
17315 {
17316 if (nmesh == 0)
17317 {
17318 my_mesh_pt = mesh_pt();
17319 }
17320 else
17321 {
17323 }
17324
17325 if (do_halos)
17326 {
17327 // How many of my nodes are halos whose non-halo counter
17328 // parts live on processor send_rank?
17329 unsigned n_nod = my_mesh_pt->nhalo_node(send_rank);
17330 for (unsigned n = 0; n < n_nod; n++)
17331 {
17332 // Generalise to variable number of values per node
17333 my_mesh_pt->halo_node_pt(send_rank, n)
17334 ->read_eqn_numbers_from_vector(receive_data, count);
17335 }
17336
17337 // Get number of halo elements whose non-halo is on
17338 // process send_rank
17340 my_mesh_pt->halo_element_pt(send_rank);
17341 unsigned nelem_halo = halo_elem_pt.size();
17342 for (unsigned e = 0; e < nelem_halo; e++)
17343 {
17346 }
17347 }
17348
17350 {
17351 // How many of my nodes are external halos whose external non-halo
17352 // counterparts live on processor send_rank?
17353 unsigned n_ext_nod = my_mesh_pt->nexternal_halo_node(send_rank);
17354 for (unsigned n = 0; n < n_ext_nod; n++)
17355 {
17356 my_mesh_pt->external_halo_node_pt(send_rank, n)
17357 ->read_eqn_numbers_from_vector(receive_data, count);
17358 }
17359
17360 // Get number of external halo elements whose external haloed
17361 // counterpart is on process send_rank
17362 unsigned next_elem_halo =
17363 my_mesh_pt->nexternal_halo_element(send_rank);
17364 for (unsigned e = 0; e < next_elem_halo; e++)
17365 {
17366 my_mesh_pt->external_halo_element_pt(send_rank, e)
17368 }
17369 }
17370
17371 } // end of loop over meshes
17372 }
17373 } // End of loop over processors
17374 }
17375
17376 //==========================================================================
17377 /// Balance the load of a (possibly non-uniformly refined) problem that has
17378 /// already been distributed, by re-distributing elements over the processors.
17379 /// Produce explicit stats of load balancing process if boolean, report_stats,
17380 /// is set to true and doc various bits of data (mainly for debugging)
17381 /// in directory specified by DocInfo object.
17382 //==========================================================================
17384 DocInfo& doc_info,
17385 const bool& report_stats,
17387 {
17388 double start_t = TimingHelpers::timer();
17389
17390 // Number of processes
17391 const unsigned n_proc = this->communicator_pt()->nproc();
17392
17393 // Don't do anything if this is a single-process job
17394 if (n_proc == 1)
17395 {
17396 if (report_stats)
17397 {
17398 std::ostringstream warn_message;
17399 warn_message << "WARNING: You've tried to load balance a problem over\n"
17400 << "only one processor: ignoring your request.\n";
17402 "Problem::load_balance()",
17404 }
17405 }
17406 // Multiple processors
17407 else
17408 {
17409 // This will only work if the problem has already been distributed
17411 {
17412 // Throw an error
17413 std::ostringstream error_stream;
17414 error_stream << "You have called Problem::load_balance()\n"
17415 << "on a non-distributed problem. This doesn't\n"
17416 << "make sense -- go distribute your problem first."
17417 << std::endl;
17418 throw OomphLibError(
17420 }
17421
17422 // Timings
17423 double t_start = 0.0;
17424 double t_metis = 0.0;
17425 double t_partition = 0.0;
17426 double t_distribute = 0.0;
17427 double t_refine = 0.0;
17428 double t_copy_solution = 0.0;
17429
17430 if (report_stats)
17431 {
17433 }
17434
17435
17436#ifdef PARANOID
17437 unsigned old_ndof = ndof();
17438#endif
17439
17440 // Store pointers to the old mesh(es) so we retain a handle
17441 //---------------------------------------------------------
17442 // to them for deletion
17443 //---------------------
17445 unsigned n_mesh = nsub_mesh();
17446 if (n_mesh == 0)
17447 {
17448 // Resize the container
17449 old_mesh_pt.resize(1);
17450 old_mesh_pt[0] = mesh_pt();
17451 }
17452 else
17453 {
17454 // Resize the container
17455 old_mesh_pt.resize(n_mesh);
17456 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17457 {
17459 }
17460 }
17461
17462
17463 // Partition the global mesh in its current state
17464 //-----------------------------------------------
17465
17466 // target_domain_for_local_non_halo_element[e] contains the number
17467 // of the domain [0,1,...,nproc-1] to which non-halo element e on THE
17468 // CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
17469 // elements is the same as in the Problem's mesh, with the halo
17470 // elements being skipped.
17472
17473 // Do any of the processors want to go through externally imposed
17474 // partitioning? If so, we'd better do it here too (even if the processor
17475 // is empty, e.g. following a restart on a larger number of procs) or
17476 // we hang.
17477 unsigned local_ntarget =
17479 unsigned global_ntarget = 0;
17482 1,
17484 MPI_MAX,
17485 Communicator_pt->mpi_comm());
17486
17487 // External prescribed partitioning
17488 if (global_ntarget > 0)
17489 {
17492 }
17493 else
17494 {
17495 // Metis does not always produce repeatable results which is
17496 // a disaster for validation runs -- this bypasses metis and
17497 // comes up with a stupid but repeatable partioning.
17499 {
17500 // Bypass METIS to perform the partitioning
17501 unsigned objective = 0;
17502 bool bypass_metis = true;
17504 this,
17505 objective,
17507 bypass_metis);
17508 }
17509 else
17510 {
17511 // Use METIS to perform the partitioning
17512 unsigned objective = 0;
17515 }
17516 }
17517
17518 if (report_stats)
17519 {
17521 }
17522
17523 // Setup map linking element with target domain
17524 std::map<GeneralisedElement*, unsigned>
17526 unsigned n_elem = mesh_pt()->nelement();
17527 unsigned count_non_halo_el = 0;
17528 for (unsigned e = 0; e < n_elem; e++)
17529 {
17531 if (!el_pt->is_halo())
17532 {
17536 }
17537 }
17538
17539 // Load balancing is equivalent to distribution so call the
17540 // appropriate "actions before". NOTE: This acts on the
17541 // current, refined, distributed, etc. problem object
17542 // before it's being wiped. This step is therefore not
17543 // a duplicate of the call below, which acts on the
17544 // new, not-yet refined, distributed etc. problem!
17546
17547 // Re-setup target domains for remaining elements (FaceElements
17548 // are likely to have been stripped out in actions_before_distribute()
17549 n_elem = mesh_pt()->nelement();
17553 for (unsigned e = 0; e < n_elem; e++)
17554 {
17556 if (!el_pt->is_halo())
17557 {
17560 }
17561 } // for (e < n_elem)
17562
17563 // Re-setup the number of sub-meshes since some of them may have
17564 // been stripped out in actions_before_distribute(), but save the
17565 // number of old sub-meshes
17566 const unsigned n_old_sub_meshes = n_mesh;
17567 n_mesh = nsub_mesh();
17568
17569 // Now get the target domains for each of the submeshes, we only
17570 // get the target domains for the nonhalo elements
17572 n_mesh);
17573 // If we have no sub-meshes then we do not need to copy the target areas
17574 // of the submeshes
17575 if (n_mesh != 0)
17576 {
17577 // Counter to copy the target domains from the global vector
17578 unsigned count_td = 0;
17579 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17580 {
17581 // Get the number of elements (considering halo elements)
17582 const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
17583 // Now copy that number of data from the global target domains
17584 for (unsigned i = 0; i < nsub_ele; i++)
17585 {
17586 // Get the element
17588 // ... and check if it is a nonhalo element
17589 if (!ele_pt->is_halo())
17590 {
17591 // Get the target domain for the current element
17592 const unsigned target_domain =
17594 // Add the target domain for the nonhalo element in the
17595 // submesh
17597 .push_back(target_domain);
17598 } // if (!ele_pt->is_halo())
17599 } // for (i < nsub_ele)
17600 } // for (imesh < n_mesh)
17601
17602#ifdef PARANOID
17603 // Check that the total number of copied data be the same as the
17604 // total number of nonhalo elements in the (sub)-mesh(es)
17605 const unsigned ntarget_domain =
17607 if (count_td != ntarget_domain)
17608 {
17609 std::ostringstream error_stream;
17611 << "The number of nonhalo elements (" << count_td
17612 << ") found in (all)\n"
17613 << "the (sub)-mesh(es) is different from the number of target "
17614 "domains\n"
17615 << "(" << ntarget_domain << ") for the nonhalo elements.\n"
17616 << "Please ensure that you called the rebuild_global_mesh() method "
17617 << "after the\npossible deletion of FaceElements in "
17618 << "actions_before_distribute()!!!\n\n";
17619 throw OomphLibError(error_stream.str(),
17620 "Problem::load_balance()",
17622 } // if (count_td != ntarget_domain)
17623#endif
17624
17625 } // if (n_mesh != 0)
17626
17627 // Check if we have different type of submeshes (unstructured
17628 // and/or structured). Identify to which type each submesh belongs.
17629 // If we have only one mesh then identify to which type that mesh
17630 // belongs.
17631
17632 // The load balancing strategy acts in the structured meshes and
17633 // then acts in the unstructured meshes
17634
17635 // Vector to temporaly store pointers to unstructured meshes
17636 // (TriangleMeshBase)
17638 std::vector<bool> is_unstructured_mesh;
17639
17640 // Flag to indicate that there are unstructured meshes as part of
17641 // the problem
17642 bool are_there_unstructured_meshes = false;
17643
17644 // We have only one mesh
17645 if (n_mesh == 0)
17646 {
17647 // Check if it is a TriangleMeshBase mesh
17649 dynamic_cast<TriangleMeshBase*>(old_mesh_pt[0]))
17650 {
17651 // Add the pointer to the unstructured meshes container
17653 // Indicate that it is an unstructured mesh
17654 is_unstructured_mesh.push_back(true);
17655 // Indicate that there are unstructured meshes as part of the
17656 // problem
17658 }
17659 else
17660 {
17661 // Add the pointer to the unstructured meshes container (null
17662 // pointer)
17664 // Indicate that it is not an unstructured mesh
17665 is_unstructured_mesh.push_back(false);
17666 }
17667 } // if (n_mesh == 0)
17668 else // We have sub-meshes
17669 {
17670 // Check which sub-meshes are unstructured meshes (work with the
17671 // old sub-meshes number)
17672 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17673 {
17674 // Is it a TriangleMeshBase mesh
17676 dynamic_cast<TriangleMeshBase*>(old_mesh_pt[i_mesh]))
17677 {
17678 // Add the pointer to the unstructured meshes container
17680 // Indicate that it is an unstructured mesh
17681 is_unstructured_mesh.push_back(true);
17682 // Indicate that there are unstructured meshes as part of the
17683 // problem
17685 }
17686 else
17687 {
17688 // Add the pointer to the unstructured meshes container (null
17689 // pointer)
17691 // Indicate that it is not an unstructured mesh
17692 is_unstructured_mesh.push_back(false);
17693 }
17694 } // for (i_mesh < n_mesh)
17695 } // else if (n_mesh == 0) // We have sub-meshes
17696
17697 // Extract data to be sent to various processors after the
17698 //--------------------------------------------------------
17699 // problem has been rebuilt/re-distributed
17700 //----------------------------------------
17701
17702 // Storage for number of data to be sent to each processor
17704
17705 // Storage for all values to be sent to all processors
17707
17708 // Start location within send_data for data to be sent to each processor
17710
17711 // Old and new domains for each base element (available for all, for
17712 // convenience)
17715
17716 // Flat-packed refinement info, labeled by id of locally
17717 // available root elements
17718 std::map<unsigned, Vector<unsigned>> flat_packed_refinement_info_for_root;
17719
17720 // Max. level of refinement
17721 unsigned max_refinement_level_overall = 0;
17722
17723 // Prepare the input for the get_data...() method, only copy the
17724 // data from the structured meshes, TreeBaseMesh meshes
17727 if (n_mesh == 0)
17728 {
17729 // Check if the mesh is an structured mesh
17730 if (!is_unstructured_mesh[0])
17731 {
17732 const unsigned nele_mesh =
17734 for (unsigned e = 0; e < nele_mesh; e++)
17735 {
17736 const unsigned target_domain =
17739 .push_back(target_domain);
17740 } // for (e < nele_mesh)
17741 } // if (!is_unstructured_mesh[0])
17742 } // if (n_mesh == 0)
17743 else
17744 {
17745 // Copy the target domains from the structured meshes only
17746 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17747 {
17748 // Check if the mesh is an structured mesh
17750 {
17751 const unsigned nele_sub_mesh =
17753 for (unsigned e = 0; e < nele_sub_mesh; e++)
17754 {
17755 const unsigned target_domain =
17758 .push_back(target_domain);
17759 } // for (e < nele_sub_mesh)
17760 } // if (!is_triangle_mesh_base[i_mesh])
17761 } // for (i_mesh < n_mesh)
17762 } // else if (n_mesh == 0)
17763
17764 // Extract data from current problem
17765 // sorted into data to be sent to various processors after
17766 // rebuilding the meshes in a load-balanced form
17769 send_n,
17770 send_data,
17775
17776 // Extract flat-packed refinement pattern
17782
17783 if (report_stats)
17784 {
17786 oomph_info << "CPU for partition calculation for roots: "
17787 << t_partition - t_metis << std::endl;
17788 }
17789
17790
17791 // Flush and delete old submeshes and null the global mesh
17792 //--------------------------------------------------------
17793 // and rebuild the new (not yet distributed, refined etc.) mesh
17794 //-------------------------------------------------------------
17795 // that will be distributed in the new, improved way determined
17796 //-------------------------------------------------------------
17797 // by METIS
17798 //---------
17800 std::max(int(n_old_sub_meshes), 1));
17801 if (n_mesh == 0)
17802 {
17805 dynamic_cast<TreeBasedRefineableMeshBase*>(old_mesh_pt[0]);
17806 if (ref_mesh_pt != 0)
17807 {
17809 ref_mesh_pt->uniform_refinement_level_when_pruned();
17810 }
17811
17812 // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17813 // then we should not delete it since the load balance strategy
17814 // requires the mesh
17815
17816 // Delete the mesh if it is not an unstructured mesh
17817 if (!is_unstructured_mesh[0])
17818 {
17819 delete old_mesh_pt[0];
17820 old_mesh_pt[0] = 0;
17821 } // if (!is_unstructured_mesh[0])
17822 } // if (n_mesh==0)
17823 else
17824 {
17825 // Loop over the number of old meshes (required to delete the
17826 // pointers of structured meshes in the old_mesh_pt structure)
17827 for (unsigned i_mesh = 0; i_mesh < n_old_sub_meshes; i_mesh++)
17828 {
17832 if (ref_mesh_pt != 0)
17833 {
17835 ref_mesh_pt->uniform_refinement_level_when_pruned();
17836 }
17837
17838 // If the mesh is an unstructured mesh (TriangleMeshBase mesh)
17839 // then we should NOT delete it since the load balance strategy
17840 // requires the mesh
17841
17842 // Delete the mesh if it is not an unstructured mesh
17844 {
17845 delete old_mesh_pt[i_mesh];
17846 old_mesh_pt[i_mesh] = 0;
17847 } // if (!is_unstructured_mesh[i_mesh])
17848
17849 } // for (i_mesh<n_mesh)
17850
17851 // Empty storage for sub-meshes
17853
17854 // Flush the storage for nodes and elements in compound mesh
17855 // (they've already been deleted in the sub-meshes)
17857
17858 // Kill
17859 delete mesh_pt();
17860 mesh_pt() = 0;
17861 } // else if (n_mesh==0)
17862
17863 bool some_mesh_has_been_pruned = false;
17864 unsigned n = pruned_refinement_level.size();
17865 for (unsigned i = 0; i < n; i++)
17866 {
17868 }
17869
17870 // (Re-)build the new mesh(es) -- this must get the problem into the
17871 // state it was in when it was first distributed!
17872 build_mesh();
17873
17874 // Has one of the meshes been pruned; if so refine to the
17875 // common refinement level
17877 {
17878 // Do actions before adapt
17880
17881 // Re-assign number of submeshes -- when this was first
17882 // set, the problem may have had face meshes that have now
17883 // disappeared.
17884 n_mesh = nsub_mesh();
17885
17886 // Now adapt meshes manually
17887 if (n_mesh == 0)
17888 {
17890 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
17891 if (ref_mesh_pt != 0)
17892 {
17893 // Get min and max refinement level
17894 unsigned local_min_ref = 0;
17895 unsigned local_max_ref = 0;
17896 ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17897
17898 // Reconcile between processors: If (e.g. following
17899 // distribution/pruning) the mesh has no elements on this
17900 // processor) then ignore its contribution to the poll of
17901 // max/min refinement levels
17903 if (ref_mesh_pt->nelement() == 0)
17904 {
17906 }
17907 int int_min_ref = 0;
17909 &int_min_ref,
17910 1,
17911 MPI_INT,
17912 MPI_MIN,
17913 Communicator_pt->mpi_comm());
17914
17915 // Overall min refinement level over all meshes
17916 unsigned min_ref = unsigned(int_min_ref);
17917
17918 // Refine as many times as required to get refinement up to
17919 // uniform refinement level after last prune
17920 unsigned nref = pruned_refinement_level[0] - min_ref;
17921 oomph_info << "Refining one-and-only mesh uniformly " << nref
17922 << " times\n";
17923 for (unsigned i = 0; i < nref; i++)
17924 {
17925 ref_mesh_pt->refine_uniformly();
17926 }
17927 }
17928 }
17929 else
17930 {
17931 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
17932 {
17935 if (ref_mesh_pt != 0)
17936 {
17937 // Get min and max refinement level
17938 unsigned local_min_ref = 0;
17939 unsigned local_max_ref = 0;
17940 ref_mesh_pt->get_refinement_levels(local_min_ref, local_max_ref);
17941
17942 // Reconcile between processors: If (e.g. following
17943 // distribution/pruning) the mesh has no elements on this
17944 // processor) then ignore its contribution to the poll of
17945 // max/min refinement levels
17947 if (ref_mesh_pt->nelement() == 0)
17948 {
17950 }
17951 int int_min_ref = 0;
17953 &int_min_ref,
17954 1,
17955 MPI_INT,
17956 MPI_MIN,
17957 Communicator_pt->mpi_comm());
17958
17959 // Overall min refinement level over all meshes
17960 unsigned min_ref = unsigned(int_min_ref);
17961
17962 // Refine as many times as required to get refinement up to
17963 // uniform refinement level after last prune
17965 oomph_info << "Refining sub-mesh " << i_mesh << " uniformly "
17966 << nref << " times\n";
17967 for (unsigned i = 0; i < nref; i++)
17968 {
17969 ref_mesh_pt->refine_uniformly();
17970 }
17971 }
17972 }
17973 // Rebuild the global mesh
17975 }
17976
17977 // Do actions after adapt
17979
17980 // Re-assign number of submeshes -- when this was first
17981 // set, the problem may have had face meshes that have now
17982 // disappeared.
17983 n_mesh = nsub_mesh();
17984 } // if (some_mesh_has_been_pruned)
17985
17986
17987 // Perform any actions before distribution but now for the new mesh
17988 // NOTE: This does NOT replicate the actions_before_distribute()
17989 // call made above for the previous mesh!
17991
17992 // Do some book-keeping
17993 //---------------------
17994
17995 // Re-assign number of submeshes -- when this was first
17996 // set, the problem may have had face meshes that have now
17997 // disappeared.
17998 n_mesh = nsub_mesh();
17999
18000 // The submeshes, if they exist, need to know their own element
18001 // domains.
18002 // NOTE: This vector only stores the target domains or the
18003 // element partition for structured meshes
18005 if (n_mesh != 0)
18006 {
18007 unsigned count = 0;
18008 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18009 {
18010 // Only work with structured meshes
18012 {
18013 // Get the number of element in the mesh
18014 const unsigned nsub_ele = mesh_pt(i_mesh)->nelement();
18016 for (unsigned e = 0; e < nsub_ele; e++)
18017 {
18020 } // for (e<nsub_elem)
18021 } // if (sub_mesh_pt!=0)
18022 } // for (i_mesh<n_mesh)
18023
18024#ifdef PARANOID
18025 const unsigned nnew_domain_for_base_element =
18028 {
18029 std::ostringstream error_stream;
18031 << "The number of READ target domains for nonhalo elements\n"
18032 << " is (" << count << "), but the number of target domains for\n"
18033 << "nonhalo elements is (" << nnew_domain_for_base_element
18034 << ")!\n";
18035 throw OomphLibError(error_stream.str(),
18036 "Problem::load_balance()",
18038 }
18039#endif
18040
18041 } // if (n_mesh!=0)
18042
18043 // Setup the map between "root" element and number in global mesh
18044 // again
18045
18046 // This map is only established for structured meshes, then we
18047 // need to check here the type of mesh
18048 if (n_mesh == 0)
18049 {
18050 // Check if the only one mesh is an stuctured mesh
18051 if (!is_unstructured_mesh[0])
18052 {
18053 const unsigned n_ele = mesh_pt()->nelement();
18056 for (unsigned e = 0; e < n_ele; e++)
18057 {
18061 } // for (e<n_ele)
18062 } // if (!is_triangle_mesh_base[0])
18063 } // if (n_mesh==0)
18064 else
18065 {
18066 // If we have submeshes then we only add those elements that
18067 // belong to structured meshes, but first compute the number of
18068 // total elements in the structured sub-meshes
18069 unsigned nglobal_element = 0;
18070 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18071 {
18072 // Check if mesh is an structured mesh
18074 {
18076 } // if (!is_triangle_mesh_base[i_mesh])
18077 } // for (i_mesh<n_mesh)
18078
18079 // Once computed the number of elements, then resize the
18080 // structure
18083 unsigned counter_base = 0;
18084 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18085 {
18086 // Check if mesh is a structured mesh
18088 {
18089 const unsigned n_ele = mesh_pt(i_mesh)->nelement();
18090 for (unsigned e = 0; e < n_ele; e++)
18091 {
18095 // Inrease the global element number
18096 counter_base++;
18097 } // for (e<n_ele)
18098 } // if (!is_triangle_mesh_base[i_mesh])
18099 } // for (i_mesh<n_mesh)
18100
18101#ifdef PARANOID
18103 {
18104 std::ostringstream error_stream;
18105 error_stream << "The number of global elements (" << nglobal_element
18106 << ") is not the same as the number of\nadded elements ("
18107 << counter_base << ") to the Base_mesh_element_pt data "
18108 << "structure!!!\n\n";
18109 throw OomphLibError(error_stream.str(),
18110 "Problem::load_balance()",
18112 } // if (counter_base != nglobal_element)
18113#endif // #ifdef PARANOID
18114
18115 } // else if (n_mesh==0)
18116
18117 // Storage for the number of face elements in the base mesh --
18118 // element is identified by number of bulk element and face index
18119 // so we can reconstruct it if and when the FaceElements have been wiped
18120 // in actions_before_distribute().
18121 // NOTE: Not really clear (any more) why this is required. Typically
18122 // FaceElements get wiped in actions_before_distribute() so
18123 // at this point there shouldn't be any of them left.
18124 // This is certainly the case in all our currently existing
18125 // test codes. However, I'm too scared to take this out
18126 // in case it does matter (we're not insisting that FaceElements
18127 // are always removed in actions_before_distribute()...).
18128 std::map<unsigned, std::map<int, unsigned>> face_element_number;
18129 unsigned n_element = mesh_pt()->nelement();
18130 for (unsigned e = 0; e < n_element; e++)
18131 {
18133 dynamic_cast<FaceElement*>(mesh_pt()->finite_element_pt(e));
18134 if (face_el_pt != 0)
18135 {
18136#ifdef PARANOID
18137 std::stringstream info;
18138 info << "================================================\n";
18139 info << "INFO: I've come across a FaceElement while \n";
18140 info << " load-balancing a problem. \n";
18141 info << "================================================\n";
18142 oomph_info << info.str() << std::endl;
18143#endif
18144 FiniteElement* bulk_elem_pt = face_el_pt->bulk_element_pt();
18146#ifdef PARANOID
18147 if (e_bulk == 0)
18148 {
18149 throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18152 }
18153#endif
18154 e_bulk -= 1;
18155 int face_index = face_el_pt->face_index();
18156 face_element_number[e_bulk][face_index] = e;
18157 }
18158 }
18159
18160 // Distribute the (sub)meshes
18161 //---------------------------
18163 if (n_mesh == 0)
18164 {
18165 // Only distribute (load balance strategy) if this is an
18166 // structured mesh
18167 if (!is_unstructured_mesh[0])
18168 {
18169#ifdef PARANOID
18170 if (mesh_pt()->nelement() != new_domain_for_base_element.size())
18171 {
18172 std::ostringstream error_stream;
18173 error_stream << "Distributing one-and-only mesh containing "
18174 << mesh_pt()->nelement() << " elements with info for "
18175 << new_domain_for_base_element.size() << std::endl;
18176 throw OomphLibError(error_stream.str(),
18179 }
18180#endif
18181
18182 if (report_stats)
18183 {
18184 oomph_info << "Distributing one and only mesh\n"
18185 << "------------------------------" << std::endl;
18186 }
18187
18188 // No pre-set distribution from restart that may leave some
18189 // processors empty so no need to overrule deletion of elements
18191
18193 new_domain_for_base_element,
18195 doc_info,
18198
18199 } // if (!is_unstructured_mesh[0])
18200
18201 } // if (n_mesh==0)
18202 else // There are submeshes, "distribute" each one separately
18203 {
18204 // Rebuild the mesh only if one of the meshes was modified
18205 bool need_to_rebuild_mesh = false;
18206 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18207 {
18208 // Perform the load balancing based on distribution in the
18209 // structured meshes only
18211 {
18212 if (report_stats)
18213 {
18214 oomph_info << "Distributing submesh " << i_mesh << " of "
18215 << n_mesh << " in total\n"
18216 << "---------------------------------------------"
18217 << std::endl;
18218 }
18219
18220 // Set the doc_info number to reflect the submesh
18221 doc_info.number() = i_mesh;
18222
18223 // No pre-set distribution from restart that may leave some
18224 // processors empty so no need to overrule deletion of elements
18227 submesh_element_partition[i_mesh],
18229 doc_info,
18232
18233 // Set the flag to rebuild the global mesh
18234 need_to_rebuild_mesh = true;
18235
18236 } // if (!is_unstructured_mesh[i_mesh])
18237
18238 } // for (i_mesh<n_mesh)
18239
18241 {
18242 // Rebuild the global mesh
18244 } // if (need_to_rebuild_mesh)
18245
18246 } // else if (n_mesh==0)
18247
18248 // Null out information associated with deleted elements
18249 unsigned n_del = deleted_element_pt.size();
18250 for (unsigned e = 0; e < n_del; e++)
18251 {
18256 }
18257
18258 // Has one of the meshes been pruned before distribution? If so
18259 // then prune here now
18261 {
18263 if (n_mesh == 0)
18264 {
18266 dynamic_cast<TreeBasedRefineableMeshBase*>(mesh_pt());
18267 if (ref_mesh_pt != 0)
18268 {
18269 ref_mesh_pt->prune_halo_elements_and_nodes(
18270 deleted_element_pt, doc_info, report_stats);
18271 }
18272 }
18273 else
18274 {
18275 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18276 {
18279 if (ref_mesh_pt != 0)
18280 {
18281 ref_mesh_pt->prune_halo_elements_and_nodes(
18282 deleted_element_pt, doc_info, report_stats);
18283 }
18284 }
18285 // Rebuild the global mesh
18287 }
18288
18289 // Null out information associated with deleted elements
18290 unsigned n_del = deleted_element_pt.size();
18291 for (unsigned e = 0; e < n_del; e++)
18292 {
18297 }
18298
18299 // Setup the map between "root" element and number in global mesh again
18301 }
18302
18303 if (report_stats)
18304 {
18306 oomph_info << "CPU for build and distribution of new mesh(es): "
18307 << t_distribute - t_partition << std::endl;
18308 }
18309
18310
18311 // Send refinement info to other processors
18312 //-----------------------------------------
18313
18314 // Storage for refinement pattern: Given ID of root element,
18315 // root_element_id, and current refinement level, level, the e-th entry in
18316 // refinement_info_for_root_elements[root_element_id][level][e] is equal
18317
18318 // to 2 if the e-th element (using the enumeration when the mesh has been
18319 // refined to the level-th level) is to be refined during the next
18320 // refinement; it's 1 if it's not to be refined.
18322
18323
18324 // Send refinement information between processors, using flat-packed
18325 // information accumulated earlier
18331
18332 // Refine each mesh based upon refinement information stored for each root
18333 //------------------------------------------------------------------------
18336
18337 if (report_stats)
18338 {
18340 oomph_info << "CPU for refinement of base mesh: "
18341 << t_refine - t_distribute << std::endl;
18342 }
18343
18344 // NOTE: The following two calls are important e.g. when
18345 // FaceElements that resize nodes are attached/detached
18346 // after/before adaptation. If we don't attach them
18347 // on the newly built/refined mesh, there isn't enough
18348 // storage for the nodal values that are sent around
18349 // (in a flat-packed format) resulting in total disaster.
18350 // So we attach them first, but then immediatly strip
18351 // them out again because the FaceElements themselves
18352 // will have been stripped out before distribution/adaptation.
18353
18354 // Do actions after adapt because we have just adapted the mesh.
18356
18357 // Now strip it back out to get problem into the same state
18358 // it was in when data to be sent was recorded.
18360
18361 // Send the stored values in each root from the old mesh into the new mesh
18362 //------------------------------------------------------------------------
18365
18366 // If there are unstructured meshes here we perform the load
18367 // balancing of those meshes
18369 {
18370 // Delete any storage of external elements and nodes
18372
18373 if (n_mesh == 0)
18374 {
18375 // Before doing the load balancing delete the mesh created at
18376 // calling build_mesh(), and restore the pointer to the old
18377 // mesh
18378
18379 // It MUST be an unstructured mesh, otherwise we should not be
18380 // here
18381 if (is_unstructured_mesh[0])
18382 {
18383 // Delete the new created mesh
18384 delete mesh_pt();
18385 // Re-assign the pointer to the old mesh
18386 this->mesh_pt() = old_mesh_pt[0];
18387 } // if (is_unstructured_mesh[0])
18388#ifdef PARANOID
18389 else
18390 {
18391 std::ostringstream error_stream;
18392 error_stream << "The only one mesh in the problem is not an "
18393 "unstructured mesh,\n"
18394 << "but the flag 'are_there_unstructures_meshes' ("
18396 << ") was turned on,\n"
18397 << "this is weird. Please check for any condition "
18398 "that may have\n"
18399 << "turned on this flag!!!!\n\n";
18400 throw OomphLibError(error_stream.str(),
18401 "Problem::load_balance()",
18403 }
18404#endif
18405
18406 unstructured_mesh_pt[0]->load_balance(
18408 } // if (n_mesh == 0)
18409 else
18410 {
18411 // Before doing the load balancing delete the meshes created
18412 // at calling build_mesh(), and restore the pointer to the
18413 // old meshes
18414 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18415 {
18417 {
18418 // Delete the new created mesh
18419 delete mesh_pt(i_mesh);
18420 // Now point it to nothing
18421 mesh_pt(i_mesh) = 0;
18422 // ... and re-assign the pointer to the old mesh
18423 this->mesh_pt(i_mesh) = old_mesh_pt[i_mesh];
18424 } // if (is_unstructured_mesh[i_mesh])
18425
18426 } // for (i_mesh<n_mesh)
18427
18428 // Empty storage for sub-meshes
18429 // flush_sub_meshes();
18430
18431 // Flush the storage for nodes and elements in compound mesh
18432 // (they've already been deleted in the sub-meshes)
18434
18435 // Now we can procede with the load balancing thing
18436 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
18437 {
18439 {
18440 // Get the number of elements in the "i_mesh" (the old one)
18441 const unsigned n_element = old_mesh_pt[i_mesh]->nelement();
18442
18443 // Perform the load balancing if there are elements in the
18444 // mesh. We check for this case because the meshes created
18445 // from face elements have been cleaned in
18446 // "actions_before_distribute()"
18448 {
18449 unstructured_mesh_pt[i_mesh]->load_balance(
18451 } // if (n_element > 0)
18452 } // if (is_unstructured_mesh[i_mesh)]
18453 } // for (i_mesh < n_mesh)
18454
18455 // Rebuild the global mesh
18457
18458 } // else if (n_mesh == 0)
18459
18460 } // if (are_there_unstructured_meshes)
18461
18462 if (report_stats)
18463 {
18465 oomph_info << "CPU for transferring solution to new mesh(es): "
18466 << t_copy_solution - t_refine << std::endl;
18467 oomph_info << "CPU for load balancing: " << t_copy_solution - t_start
18468 << std::endl;
18469 }
18470
18471 // Do actions after distribution
18473
18474 // Re-assign equation numbers
18475#ifdef PARANOID
18476 unsigned n_dof = assign_eqn_numbers();
18477#else
18479#endif
18480
18481 if (report_stats)
18482 {
18484 << "Total number of elements on this processor after load balance: "
18485 << mesh_pt()->nelement() << std::endl;
18486
18487 oomph_info << "Number of non-halo elements on this processor after "
18488 "load balance: "
18489 << mesh_pt()->nnon_halo_element() << std::endl;
18490 }
18491
18492#ifdef PARANOID
18493 if (n_dof != old_ndof)
18494 {
18495 std::ostringstream error_stream;
18497 << "Number of dofs in load_balance() has changed from " << old_ndof
18498 << " to " << n_dof << "\n"
18499 << "Check that you've implemented any necessary "
18500 "actions_before/after\n"
18501 << "adapt/distribute functions, e.g. to pin redundant pressure dofs"
18502 << " etc.\n";
18503 throw OomphLibError(
18505 }
18506#endif
18507 }
18508
18509 // Finally synchronise all dofs to allow halo check to pass
18511
18512 double end_t = TimingHelpers::timer();
18513 oomph_info << "Time for load_balance() [sec] : " << end_t - start_t
18514 << std::endl;
18515 }
18516
18517
18518 //==========================================================================
18519 /// Send refinement information between processors
18520 //==========================================================================
18524 const unsigned& max_refinement_level_overall,
18527 {
18528 // Number of processes etc.
18529 const int n_proc = this->communicator_pt()->nproc();
18530 const int my_rank = this->communicator_pt()->my_rank();
18531
18532 // Make space
18535
18536 // Make space for list of domains that the refinement info
18537 // is to be forwarded to
18538 std::map<unsigned, Vector<unsigned>> halo_domain_of_haloed_base_element;
18539
18540 // Find out haloed elements in new, redistributed problem
18541 //-------------------------------------------------------
18542
18543 // halo_domains[e][j] = j-th halo domain associated with (haloed) element e
18544 std::map<unsigned, Vector<unsigned>> halo_domains;
18545
18546 // Loop over sub meshes
18547 unsigned n_sub_mesh = nsub_mesh();
18548 unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
18549 for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
18550 {
18551 // Choose the right mesh
18552 Mesh* my_mesh_pt = 0;
18553 if (n_sub_mesh == 0)
18554 {
18555 my_mesh_pt = mesh_pt();
18556 }
18557 else
18558 {
18560 }
18561
18562 // Only work with structured meshes
18564 dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
18565 if (!(sub_mesh_pt != 0))
18566 {
18567 // Loop over processors to find haloed elements -- need to
18568 // send their refinement patterns processors that hold their
18569 // halo counterparts!
18570 for (int p = 0; p < n_proc; p++)
18571 {
18573 my_mesh_pt->haloed_element_pt(p);
18574 unsigned nhaloed = haloed_elem_pt.size();
18575 for (unsigned h = 0; h < nhaloed; h++)
18576 {
18577 // This element must send its refinement information to processor p
18579#ifdef PARANOID
18580 if (e == 0)
18581 {
18582 throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
18585 }
18586#endif
18587 e -= 1;
18588 halo_domains[e].push_back(p);
18589 }
18590 }
18591 } // if (!(sub_mesh_pt!=0))
18592 } // for (i_mesh<max_mesh)
18593
18594 // Accumulate relevant flat-packed refinement data to be sent to
18595 //--------------------------------------------------------------
18596 // various processors
18597 //-------------------
18598
18599 // Map to accumulate unsigned data to be sent to each processor
18600 // (map for sparsity)
18601 std::map<unsigned, Vector<unsigned>> data_for_proc;
18602
18603 // Number of base elements to be sent to specified domain
18605
18606 // Total number of entries in send vector
18607 unsigned count = 0;
18608
18609 // Loop over all base elements
18610 //----------------------------
18611 for (unsigned e = 0; e < n_base_element; e++)
18612 {
18613 // Is it one of mine (i.e. was it a non-halo element on this
18614 //----------------------------------------------------------
18615 // processor before re-distribution, and do I therefore hold
18616 //----------------------------------------------------------
18617 // refinement information for it)?
18618 //--------------------------------
18620 {
18621 // Where does it go?
18623
18624 // Keep counting
18626
18627 // If it stays local, deal with it here
18628 if (int(new_domain) == my_rank)
18629 {
18630 // Record on which other procs/domains the refinement info for
18631 // this element is required because it's haloed.
18632 unsigned nhalo = halo_domains[e].size();
18634 for (unsigned j = 0; j < nhalo; j++)
18635 {
18637 }
18638
18639 // Provide storage for refinement pattern
18642
18643#ifdef PARANOID
18644 // Get number of additional data sent for check
18645 unsigned n_additional_data =
18647#endif
18648
18649 // Get number of tree nodes
18651
18652 // Counter for entries to be processed locally
18653 unsigned local_count = 1; // (have already processed zero-th entry)
18654
18655 // Loop over levels and number of nodes in tree
18656 for (unsigned level = 0; level < max_refinement_level_overall;
18657 level++)
18658 {
18659 for (unsigned ee = 0; ee < n_tree_nodes; ee++)
18660 {
18661 // Element exists at this level
18663 {
18664 local_count++;
18665
18666 // Element should be refined
18668 {
18669 refinement_info_for_root_elements[e][level].push_back(2);
18670 local_count++;
18671 }
18672 // Element should not be refined
18673 else
18674 {
18675 refinement_info_for_root_elements[e][level].push_back(1);
18676 local_count++;
18677 }
18678 }
18679 // Element does not exist at this level
18680 else
18681 {
18682 refinement_info_for_root_elements[e][level].push_back(0);
18683 local_count++;
18684 }
18685 }
18686 }
18687
18688#ifdef PARANOID
18690 {
18691 std::stringstream error_message;
18692 error_message << "Number of additional data: " << n_additional_data
18693 << " doesn't match that actually send: "
18694 << local_count << std::endl;
18695 throw OomphLibError(error_message.str(),
18698 }
18699#endif
18700 }
18701 // Element in question is not one of mine so prepare for sending
18702 //--------------------------------------------------------------
18703 else
18704 {
18705 // Make space
18707 unsigned n_additional_data =
18710 2);
18711
18712 // Keep counting
18713 count += n_additional_data + 2;
18714
18715 // Add base element number
18716 data_for_proc[new_domain].push_back(e);
18717
18718#ifdef PARANOID
18719 // Add number of flat-packed instructions to follow
18721#endif
18722
18723 // Add flat packed refinement data
18724 for (unsigned j = 0; j < n_additional_data; j++)
18725 {
18726 data_for_proc[new_domain].push_back(
18728 }
18729 }
18730 }
18731 }
18732
18733
18734 // Now do the actual send/receive
18735 //-------------------------------
18736
18737 // Storage for number of data to be sent to each processor
18739
18740 // Storage for all values to be sent to all processors
18742 send_data.reserve(count);
18743
18744 // Start location within send_data for data to be sent to each processor
18746
18747 // Loop over all processors
18748 for (int rank = 0; rank < n_proc; rank++)
18749 {
18750 // Set the offset for the current processor
18752
18753 // Don't bother to do anything if the processor in the loop is the
18754 // current processor
18755 if (rank != my_rank)
18756 {
18757 // Record how many base elements are to be sent
18759
18760 // Add data
18761 unsigned n_data = data_for_proc[rank].size();
18762 for (unsigned j = 0; j < n_data; j++)
18763 {
18764 send_data.push_back(data_for_proc[rank][j]);
18765 }
18766 }
18767
18768 // Find the number of data added to the vector
18770 }
18771
18772 // Storage for the number of data to be received from each processor
18774
18775 // Now send numbers of data to be sent between all processors
18776 MPI_Alltoall(&send_n[0],
18777 1,
18778 MPI_INT,
18779 &receive_n[0],
18780 1,
18781 MPI_INT,
18782 this->communicator_pt()->mpi_comm());
18783
18784 // We now prepare the data to be received
18785 // by working out the displacements from the received data
18787 int receive_data_count = 0;
18788 for (int rank = 0; rank < n_proc; ++rank)
18789 {
18790 // Displacement is number of data received so far
18793 }
18794
18795 // Now resize the receive buffer for all data from all processors
18796 // Make sure that it has a size of at least one
18797 if (receive_data_count == 0)
18798 {
18800 }
18802
18803 // Make sure that the send buffer has size at least one
18804 // so that we don't get a segmentation fault
18805 if (send_data.size() == 0)
18806 {
18807 send_data.resize(1);
18808 }
18809
18810 // Now send the data between all the processors
18812 &send_n[0],
18815 &receive_data[0],
18816 &receive_n[0],
18819 this->communicator_pt()->mpi_comm());
18820
18821
18822 // Now use the received data to update
18823 //-----------------------------------
18824 for (int send_rank = 0; send_rank < n_proc; send_rank++)
18825 {
18826 // Don't bother to do anything for the processor corresponding to the
18827 // current processor or if no data were received from this processor
18828 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
18829 {
18830 // Counter for the data within the large array
18832
18833 // Loop over base elements
18834 unsigned nbase_element = receive_data[count];
18835 count++;
18836 for (unsigned b = 0; b < nbase_element; b++)
18837 {
18838 // Get base element number
18840 count++;
18841
18842 // Record on which other procs/domains the refinement info for
18843 // this element is required because it's haloed.
18846 for (unsigned j = 0; j < nhalo; j++)
18847 {
18850 }
18851
18852 // Provide storage for refinement pattern
18855
18856 // Get number of flat-packed instructions to follow
18857 // (only used for check)
18858#ifdef PARANOID
18860 count++;
18861
18862 // Counter for number of additional data (validation only)
18863 unsigned check_count = 0;
18864#endif
18865
18866 // Get number of tree nodes
18867 unsigned n_tree_nodes = receive_data[count];
18868 count++;
18869
18870#ifdef PARANOID
18871 check_count++;
18872#endif
18873
18874 // Loop over levels and number of nodes in tree
18875 for (unsigned level = 0; level < max_refinement_level_overall;
18876 level++)
18877 {
18878 for (unsigned e = 0; e < n_tree_nodes; e++)
18879 {
18880 // Element exists at this level
18881 if (receive_data[count] == 1)
18882 {
18883 count++;
18884
18885#ifdef PARANOID
18886 check_count++;
18887#endif
18888
18889 // Element should be refined
18890 if (receive_data[count] == 1)
18891 {
18893 .push_back(2);
18894 count++;
18895
18896#ifdef PARANOID
18897 check_count++;
18898#endif
18899 }
18900 // Element should not be refined
18901 else
18902 {
18904 .push_back(1);
18905 count++;
18906
18907#ifdef PARANOID
18908 check_count++;
18909#endif
18910 }
18911 }
18912 // Element does not exist at this level
18913 else
18914 {
18916 .push_back(0);
18917 count++;
18918
18919#ifdef PARANOID
18920 check_count++;
18921#endif
18922 }
18923 }
18924 }
18925
18926#ifdef PARANOID
18928 {
18929 std::stringstream error_message;
18930 error_message << "Number of additional data: " << n_additional_data
18931 << " doesn't match that actually send: "
18932 << check_count << std::endl;
18933 throw OomphLibError(error_message.str(),
18936 }
18937#endif
18938 }
18939 }
18940 }
18941
18942
18943 // Now send the fully assembled refinement info to halo elements
18944 //---------------------------------------------------------------
18945 {
18946 // Accumulate data to be sent
18947 //---------------------------
18948
18949 // Map to accumulate data to be sent to other procs
18950 // (map for sparsity)
18951 std::map<unsigned, Vector<unsigned>> data_for_proc;
18952
18953 // Number of base elements to be sent to specified domain
18955
18956 // Loop over all haloed root elements and find out which
18957 // processors they have haloes on
18958 for (std::map<unsigned, Vector<unsigned>>::iterator it =
18961 it++)
18962 {
18963 // Get base element number
18964 unsigned base_element_number = (*it).first;
18965
18966 // Loop over target domains
18967 Vector<unsigned> domains = (*it).second;
18968 unsigned nd = domains.size();
18969 for (unsigned jd = 0; jd < nd; jd++)
18970 {
18971 // Actual number of domain
18972 unsigned d = domains[jd];
18973
18974 // Keep counting number of base elemements for domain
18976
18977 // Write base element number
18978 data_for_proc[d].push_back(base_element_number);
18979
18980 // Write refinement info in flat-packed form
18981 for (unsigned level = 0; level < max_refinement_level_overall;
18982 level++)
18983 {
18984 // Number of entries at each level
18985 unsigned n =
18987 .size();
18988 data_for_proc[d].push_back(n);
18989 for (unsigned j = 0; j < n; j++)
18990 {
18991 data_for_proc[d].push_back(
18993 [j]);
18994 }
18995 }
18996 }
18997 }
18998
18999
19000 // Do the actual send
19001 //-------------------
19002
19003 // Storage for number of data to be sent to each processor
19005
19006 // Storage for all values to be sent to all processors
19008 send_data.reserve(count);
19009
19010 // Start location within send_data for data to be sent to each processor
19012
19013 // Loop over all processors
19014 for (int rank = 0; rank < n_proc; rank++)
19015 {
19016 // Set the offset for the current processor
19018
19019 // Don't bother to do anything if the processor in the loop is the
19020 // current processor
19021 if (rank != my_rank)
19022 {
19023 // Record how many base elements are to be sent
19025
19026 // Add data
19027 unsigned n_data = data_for_proc[rank].size();
19028 for (unsigned j = 0; j < n_data; j++)
19029 {
19030 send_data.push_back(data_for_proc[rank][j]);
19031 }
19032 }
19033 // Find the number of data added to the vector
19035 }
19036
19037 // Storage for the number of data to be received from each processor
19039
19040 // Now send numbers of data to be sent between all processors
19041 MPI_Alltoall(&send_n[0],
19042 1,
19043 MPI_INT,
19044 &receive_n[0],
19045 1,
19046 MPI_INT,
19047 this->communicator_pt()->mpi_comm());
19048
19049 // We now prepare the data to be received
19050 // by working out the displacements from the received data
19052 int receive_data_count = 0;
19053 for (int rank = 0; rank < n_proc; ++rank)
19054 {
19055 // Displacement is number of data received so far
19058 }
19059
19060 // Now resize the receive buffer for all data from all processors
19061 // Make sure that it has a size of at least one
19062 if (receive_data_count == 0)
19063 {
19065 }
19067
19068 // Make sure that the send buffer has size at least one
19069 // so that we don't get a segmentation fault
19070 if (send_data.size() == 0)
19071 {
19072 send_data.resize(1);
19073 }
19074
19075 // Now send the data between all the processors
19077 &send_n[0],
19080 &receive_data[0],
19081 &receive_n[0],
19084 this->communicator_pt()->mpi_comm());
19085
19086
19087 // Now use the received data
19088 //------------------------
19089 for (int send_rank = 0; send_rank < n_proc; send_rank++)
19090 {
19091 // Don't bother to do anything for the processor corresponding to the
19092 // current processor or if no data were received from this processor
19093 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
19094 {
19095 // Counter for the data within the large array
19097
19098 // Read number of base elements
19099 unsigned nbase_element = receive_data[count];
19100 count++;
19101
19102 for (unsigned e = 0; e < nbase_element; e++)
19103 {
19104 // Read base element number
19106 count++;
19107
19108 // Provide storage for refinement pattern
19111
19112 // Read refinement info in flat-packed form
19113 for (unsigned level = 0; level < max_refinement_level_overall;
19114 level++)
19115 {
19116 // Read number of entries at each level
19117 unsigned n = receive_data[count];
19118 count++;
19119
19120 // Read entries
19121 for (unsigned j = 0; j < n; j++)
19122 {
19124 .push_back(receive_data[count]);
19125 count++;
19126 }
19127 }
19128 }
19129 }
19130 }
19131 }
19132 }
19133
19134 //==========================================================================
19135 /// Load balance helper routine: Send data to other
19136 /// processors during load balancing.
19137 /// - send_n: Input, number of data to be sent to each processor
19138 /// - send_data: Input, storage for all values to be sent to all processors
19139 /// - send_displacement: Input, start location within send_data for data to
19140 /// be sent to each processor
19141 //==========================================================================
19146 {
19147 // Communicator info
19148 OomphCommunicator* comm_pt = this->communicator_pt();
19149 const int n_proc = comm_pt->nproc();
19150
19151 // Storage for the number of data to be received from each processor
19153
19154 // Now send numbers of data to be sent between all processors
19155 MPI_Alltoall(&send_n[0],
19156 1,
19157 MPI_INT,
19158 &receive_n[0],
19159 1,
19160 MPI_INT,
19161 this->communicator_pt()->mpi_comm());
19162
19163 // We now prepare the data to be received
19164 // by working out the displacements from the received data
19166 int receive_data_count = 0;
19167 for (int rank = 0; rank < n_proc; ++rank)
19168 {
19169 // Displacement is number of data received so far
19172 }
19173
19174 // Now resize the receive buffer for all data from all processors
19175 // Make sure that it has a size of at least one
19176 if (receive_data_count == 0)
19177 {
19179 }
19181
19182 // Make sure that the send buffer has size at least one
19183 // so that we don't get a segmentation fault
19184 if (send_data.size() == 0)
19185 {
19186 send_data.resize(1);
19187 }
19188
19189 // Now send the data between all the processors
19191 &send_n[0],
19193 MPI_DOUBLE,
19194 &receive_data[0],
19195 &receive_n[0],
19197 MPI_DOUBLE,
19198 this->communicator_pt()->mpi_comm());
19199
19200 unsigned el_count = 0;
19201
19202 // Only do each node once
19204
19205 // Now use the received data to update the halo nodes
19206 for (int send_rank = 0; send_rank < n_proc; send_rank++)
19207 {
19208 // Don't bother to do anything if no data were received from this
19209 // processor
19210 // NOTE: We do have to loop over our own processor number to process
19211 // the data locally.
19212 if (receive_n[send_rank] != 0)
19213 {
19214 // Counter for the data within the large array
19216
19217 // How many batches are there for current rank
19218 unsigned nbatch = unsigned(receive_data[count]);
19219 count++;
19220
19221 // Loop over batches (containing leaves associated with root elements)
19222 for (unsigned b = 0; b < nbatch; b++)
19223 {
19224 // How many elements were received for this batch?
19225 unsigned nel = unsigned(receive_data[count]);
19226 count++;
19227
19228 // Get the unique base/root element number of this batch
19229 // in unrefined mesh
19231 count++;
19232
19233 // Get pointer to base/root element from reverse lookup scheme
19235
19236 // Vector for pointers to associated elements in batch
19238
19239 // Is it a refineable element?
19241 dynamic_cast<RefineableElement*>(root_el_pt);
19242 if (ref_root_el_pt != 0)
19243 {
19244 // Get all leaves associated with this base/root element
19246 ref_root_el_pt->tree_pt()->stick_leaves_into_vector(
19248
19249 // How many leaves are there?
19250 unsigned n_leaf = all_leaf_nodes_pt.size();
19251
19252#ifdef PARANOID
19253 if (n_leaf != nel)
19254 {
19255 std::ostringstream error_message;
19256 error_message
19257 << "Number of leaves: " << n_leaf << " "
19258 << " doesn't match number of elements sent in batch: " << nel
19259 << "\n";
19260 throw OomphLibError(error_message.str(),
19263 }
19264#endif
19265
19266 // Loop over batch of elements associated with this base/root
19267 // element
19268 batch_el_pt.resize(n_leaf);
19269 for (unsigned e = 0; e < n_leaf; e++)
19270 {
19271 batch_el_pt[e] = all_leaf_nodes_pt[e]->object_pt();
19272 }
19273 }
19274 // Not refineable -- the batch contains just the root element itself
19275 else
19276 {
19277#ifdef PARANOID
19278 if (1 != nel)
19279 {
19280 std::ostringstream error_message;
19281 error_message
19282 << "Non-refineable root element should only be associated with"
19283 << " one element but nel=" << nel << "\n";
19284 throw OomphLibError(error_message.str(),
19287 }
19288#endif
19289 batch_el_pt.push_back(root_el_pt);
19290 }
19291
19292 // Now loop over all elements in batch
19293 for (unsigned e = 0; e < nel; e++)
19294 {
19296 el_count++;
19297
19298 // FE?
19299 FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19300 if (fe_pt != 0)
19301 {
19302 // Loop over nodes
19303 unsigned nnod = fe_pt->nnode();
19304 for (unsigned j = 0; j < nnod; j++)
19305 {
19306 Node* nod_pt = fe_pt->node_pt(j);
19307 if (!node_done[send_rank][nod_pt])
19308 {
19309 node_done[send_rank][nod_pt] = true;
19310
19311
19312 // Read number of values (as double) to allow for resizing
19313 // before read (req'd in case we store data that
19314 // got introduced by attaching FaceElements to bulk)
19315 unsigned nval = unsigned(receive_data[count]);
19316 count++;
19317
19318#ifdef PARANOID
19319 // Does the size match?
19320 if (nval < nod_pt->nvalue())
19321 {
19322 std::ostringstream error_message;
19323 error_message
19324 << "Node has more values, namely " << nod_pt->nvalue()
19325 << ", than we're about to receive, namely " << nval
19326 << ". Something's wrong!\n";
19327 throw OomphLibError(error_message.str(),
19330 }
19331#endif
19332
19333
19334#ifdef PARANOID
19335 // Check if it's been sent as a boundary node
19337 count++;
19338#endif
19339
19340 // Check if it's actually a boundary node
19342 dynamic_cast<BoundaryNodeBase*>(nod_pt);
19343 if (bnod_pt != 0)
19344 {
19345#ifdef PARANOID
19346 // Check if local and received status are consistent
19347 if (is_boundary_node != 1)
19348 {
19349 std::ostringstream error_message;
19350 error_message << "Local node is boundary node but "
19351 "information sent is\n"
19352 << "for non-boundary node\n";
19353 throw OomphLibError(error_message.str(),
19356 }
19357#endif
19358
19359 // Do we have entries in the map?
19360 unsigned n_entry = unsigned(receive_data[count]);
19361 count++;
19362 if (n_entry > 0)
19363 {
19364 // Create storage, if it doesn't already exist, for the
19365 // map that will contain the position of the first entry
19366 // of this face element's additional values,
19367 if (
19368 bnod_pt
19369 ->index_of_first_value_assigned_by_face_element_pt() ==
19370 0)
19371 {
19372 bnod_pt
19373 ->index_of_first_value_assigned_by_face_element_pt() =
19374 new std::map<unsigned, unsigned>;
19375 }
19376
19377 // Get pointer to the map of indices associated with
19378 // additional values created by face elements
19379 std::map<unsigned, unsigned>* map_pt =
19380 bnod_pt
19381 ->index_of_first_value_assigned_by_face_element_pt();
19382
19383 // Loop over number of entries in map
19384 for (unsigned i = 0; i < n_entry; i++)
19385 {
19386 // Read out pairs...
19387 unsigned first = unsigned(receive_data[count]);
19388 count++;
19389 unsigned second = unsigned(receive_data[count]);
19390 count++;
19391
19392 // ...and assign
19393 (*map_pt)[first] = second;
19394 }
19395 }
19396 }
19397#ifdef PARANOID
19398 // Not a boundary node
19399 else
19400 {
19401 // Check if local and received status are consistent
19402 if (is_boundary_node != 0)
19403 {
19404 std::ostringstream error_message;
19405 error_message << "Local node is not a boundary node but "
19406 "information \n"
19407 << "sent is for boundary node.\n";
19408 throw OomphLibError(error_message.str(),
19411 }
19412 }
19413#endif
19414
19415 // Do we have to resize? This can happen if node was
19416 // resized (due to a FaceElement that hasn't been attached
19417 // yet here) when the send data was written. If so make space
19418 // for the data here
19419 if (nval > nod_pt->nvalue())
19420 {
19421 nod_pt->resize(nval);
19422 }
19423
19424 // Now read the actual values
19425 nod_pt->read_values_from_vector(receive_data, count);
19426 }
19427 }
19428 }
19429
19430 // Now add internal data
19432 }
19433 }
19434 }
19435 }
19436
19437 // Now that this is done, we need to synchronise dofs to get
19438 // the halo element and node values correct
19439 bool do_halos = true;
19440 bool do_external_halos = false;
19441 this->synchronise_dofs(do_halos, do_external_halos);
19442
19443 // Now rebuild global mesh if required
19444 unsigned n_mesh = nsub_mesh();
19445 if (n_mesh != 0)
19446 {
19447 bool do_halos = false;
19448 bool do_external_halos = true;
19449 this->synchronise_dofs(do_halos, do_external_halos);
19451 }
19452 }
19453
19454
19455 //==========================================================================
19456 /// Load balance helper routine: Get data to be sent to other
19457 /// processors during load balancing and other information about
19458 /// re-distribution.
19459 /// - target_domain_for_local_non_halo_element: Input, generated by METIS.
19460 /// target_domain_for_local_non_halo_element[e] contains the number
19461 /// of the domain [0,1,...,nproc-1] to which non-halo element e on THE
19462 /// CURRENT PROCESSOR ONLY has been assigned. The order of the non-halo
19463 /// elements is the same as in the Problem's mesh, with the halo
19464 /// elements being skipped.
19465 /// - send_n: Output, number of data to be sent to each processor
19466 /// - send_data: Output, storage for all values to be sent to all processors
19467 /// - send_displacement: Output, start location within send_data for data to
19468 /// be sent to each processor
19469 /// - max_refinement_level_overall: Output, max. refinement level of any
19470 /// element
19471 //==========================================================================
19480 {
19481 // Communicator info
19482 OomphCommunicator* comm_pt = this->communicator_pt();
19483 const int n_proc = comm_pt->nproc();
19484 const int my_rank = this->communicator_pt()->my_rank();
19485
19486 //------------------------------------------------------------------------
19487 // Overall strategy: Loop over all elements (in structured meshes),
19488 // identify their corresponding root elements and move all associated
19489 // leaves together, collecting the leaves in batches.
19490 // ------------------------------------------------------------------------
19491
19492 // Map to store whether the root element has been visited yet
19493 std::map<RefineableElement*, bool> root_el_done;
19494
19495#ifdef PARANOID
19496
19497 // Map for checking if all elements associated with same root
19498 // have the same target processor
19499 std::map<RefineableElement*, unsigned> target_plus_one_for_root;
19500
19501#endif
19502
19503 // Storage for maximum refinement level
19504 unsigned max_refinement_level = 0;
19505
19506 // Storage for (vector of) elements associated with target domain
19507 // (stored in map for sparsity): element_for_processor[d][e] is pointer
19508 // to e-th element that's supposed to move onto processor (domain) d.
19509 std::map<unsigned, Vector<GeneralisedElement*>> element_for_processor;
19510
19511 // Storage for the number of elements in a specified batch of leaf
19512 // elements, all of which are associated with the same root/base element:
19513 // nelement_batch_for_processor[d][j] is the number of (leaf)
19514 // elements (all associated with the same root) to be moved together to
19515 // domain/processor d, in the j-th batch of elements.
19516 std::map<unsigned, Vector<unsigned>> nelement_batch_for_processor;
19517
19518 // Storage for the unique number of the root element (in the unrefined
19519 // base mesh) whose leaves are moved together in a batch:
19520 // base_element_for_element_batch_for_processo[d][j] is the number of
19521 // unique number of the root element (in the unrefined
19522 // base mesh) of all leaf elements (associated with that root),
19523 // to be moved together to domain/processor d, in the j-th batch of
19524 // elements.
19525 std::map<unsigned, Vector<unsigned>>
19527
19528 // Record old and new domains for non-halo root elements (will be
19529 // communicated globally). Initialise to -1 so we can use max
19530 // to extract the right one via MPI_Allreduce.
19531 // NOTE: We communicate these globally to facilitate distribution
19532 // of refinement pattern. While the data itself can be
19533 // sent point-to-point for non-halo elements,
19534 // mesh refinement information also needs to be sent for
19535 // halo elements which aren't known yet.
19539
19540 // Loop over all non-halo elements on current processor and identify roots
19541 // -------------------------------------------------------------------
19542 // All leaf elements in associated tree (must!) get moved together
19543 //----------------------------------------------------------------
19544 unsigned count_non_halo_el = 0;
19545 // Get the number of submeshs, if there are no submeshes, then
19546 // increase the counter so that the loop below also work for the only
19547 // one mesh in the problem
19548 unsigned n_mesh = nsub_mesh();
19549 if (n_mesh == 0)
19550 {
19551 n_mesh = 1;
19552 }
19553 // We need to know if there are structure meshes (with elements) as
19554 // part of the problem in order to perform (or not) the proper
19555 // communications
19556 bool are_there_structured_meshes = false;
19557 // Go for the nonhalo elements only in the TreeBaseMeshes
19558 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
19559 {
19560 // Only work with structured meshes
19562 dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
19563 if (!(sub_mesh_pt != 0))
19564 {
19565 const unsigned nele = mesh_pt(i_mesh)->nelement();
19566 if (nele > 0)
19567 {
19568 // Change the flag to indicate that there are structured meshes
19569 // (with elements, because we may have meshes with face
19570 // elements and therefore zero elements at this point)
19572 }
19573
19574 for (unsigned e = 0; e < nele; e++)
19575 {
19577 if (!el_pt->is_halo())
19578 {
19579 // New non-halo: Where is this element supposed to go to?
19580 //-------------------------------------------------------
19581 unsigned target_domain =
19583
19584 // Bump up counter for non-halo elements
19586
19587 // Is it a root element? (It is, trivially, if it's not refineable)
19588 //------------------------------------------------------------------
19590 dynamic_cast<RefineableElement*>(el_pt);
19591 if (ref_el_pt == 0)
19592 {
19593 // Not refineable so add element itself
19595
19596 // Number of elements associated with this root/base
19597 // element (just the element itself)
19599
19600 // This is the unique base/root element number in unrefined mesh
19603#ifdef PARANOID
19605 {
19606 throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
19609 }
19610#endif
19613 .push_back(element_number_in_base_mesh);
19614
19615 /// Where do I come from, where do I go to?
19617 my_rank;
19620 } // if (ref_el_pt==0)
19621 // It's not a root element so we package its leaves into a batch
19622 //--------------------------------------------------------------
19623 // of elements
19624 //------------
19625 else
19626 {
19627 // Get the root element
19628 RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
19629
19630 // Has this root been visited yet?
19632 {
19633 // Now we've done it
19634 root_el_done[root_el_pt] = true;
19635
19636 // Unique number of root element in base mesh
19639#ifdef PARANOID
19641 {
19642 throw OomphLibError(
19643 "Base_mesh_element_number_plus_one[...]=0",
19646 }
19647#endif
19649
19650 /// Where do I come from, where do I go to?
19652 my_rank;
19655
19656#ifdef PARANOID
19657 // Store target domain associated with this root element
19658 // (offset by one) to allow checking that all elements
19659 // with the same root move to the same processor
19661#endif
19662
19663 // Package all leaves into batch of elements
19665 root_el_pt->tree_pt()->stick_leaves_into_vector(
19667
19668 // Number of leaves
19669 unsigned n_leaf = all_leaf_nodes_pt.size();
19670
19671 // Number of elements associated with this root/base element
19672 // (all the leaves)
19674
19675 // Store the unique base/root element number in unrefined mesh
19677 .push_back(element_number_in_base_mesh);
19678
19679 // Loop over leaves
19680 for (unsigned i_leaf = 0; i_leaf < n_leaf; i_leaf++)
19681 {
19682 // Add element object at leaf
19684 all_leaf_nodes_pt[i_leaf]->object_pt();
19686
19687 // Monitor/update maximum refinement level
19688 unsigned level = all_leaf_nodes_pt[i_leaf]->level();
19689 if (level > max_refinement_level)
19690 {
19691 max_refinement_level = level;
19692 }
19693 }
19694 }
19695
19696#ifdef PARANOID
19697 // Root element has already been visited
19698 else
19699 {
19700 // We don't have to do anything with this element since it's
19701 // already been processed earlier, but check that it's scheduled
19702 // to go onto the same processor as its root.
19704 {
19705 std::ostringstream error_message;
19706 error_message
19707 << "All elements associated with same root must have "
19708 << "same target. during load balancing\n";
19709 throw OomphLibError(error_message.str(),
19712 }
19713 }
19714#endif
19715 } // else if (ref_el_pt==0)
19716 } // if (!ele_pt->is_halo())
19717 } // for (e < nele)
19718 } // if (!(sub_mesh_pt!=0))
19719 } // for (i_mesh < n_mesh)
19720
19721#ifdef PARANOID
19722 // Have we processed all target domains?
19724 {
19725 std::ostringstream error_message;
19726 error_message
19727 << "Have processed " << count_non_halo_el << " of "
19729 << " target domains for local non-halo elelemts. \n "
19730 << "Very Odd -- we do (now) strip out the information for elements\n"
19731 << "that are removed in actions_before_distribute()...\n";
19732 throw OomphLibError(
19733 error_message.str(), OOMPH_CURRENT_FUNCTION, OOMPH_EXCEPTION_LOCATION);
19734 }
19735#endif
19736
19737 // Determine max. refinement level and origin/destination scheme
19738 // -------------------------------------------------------------
19739 // for all root/base elements
19740 // --------------------------
19741
19742 // Allreduce to work out max max refinement level across all processors
19744
19745 // Only perform this communications if necessary (it means if there
19746 // are structured meshes as part of the problem)
19748 {
19749 MPI_Allreduce(&max_refinement_level,
19751 1,
19753 MPI_MAX,
19754 comm_pt->mpi_comm());
19755 } // if (are_there_structured_meshes)
19756
19757 // Allreduce to tell everybody about the original and new domains
19758 // for root elements
19760
19761 // Only perform this communications if necessary (it means if there
19762 // are structured meshes as part of the problem)
19764 {
19768 MPI_INT,
19769 MPI_MAX,
19770 comm_pt->mpi_comm());
19771 } // if (are_there_structured_meshes)
19772
19774 // Only perform this communications if necessary (it means if there
19775 // are structured meshes as part of the problem)
19777 {
19781 MPI_INT,
19782 MPI_MAX,
19783 comm_pt->mpi_comm());
19784 } // if (are_there_structured_meshes)
19785
19786 // Copy across (after optional sanity check)
19789 for (unsigned j = 0; j < n_base_element; j++)
19790 {
19791#ifdef PARANOID
19793 {
19794 std::ostringstream error_message;
19795 error_message << "Old domain for base element " << j << ": "
19797 << "or its incarnation as refineable el: "
19798 << dynamic_cast<RefineableElement*>(
19800 << " which is of type "
19801 << typeid(*Base_mesh_element_pt[j]).name()
19802 << " does not\n"
19803 << "appear to have been assigned by any processor\n";
19804 throw OomphLibError(error_message.str(),
19807 }
19808#endif
19810#ifdef PARANOID
19812 {
19813 std::ostringstream error_message;
19814 error_message << "New domain for base element " << j
19815 << "which is of type "
19816 << typeid(*Base_mesh_element_pt[j]).name()
19817 << " does not\n"
19818 << "appear to have been assigned by any processor\n";
19819 throw OomphLibError(error_message.str(),
19822 }
19823#endif
19825 }
19826
19827
19828 // Loop over all processors and accumulate data to be sent
19829 //--------------------------------------------------------
19830 send_data.clear();
19831
19832 // Only do each node once (per processor!)
19834
19835 // Loop over all processors. NOTE: We include current processor
19836 // since we have to refine local elements too -- store their data
19837 // in same data structure as the one used for off-processor elements.
19838 for (int rank = 0; rank < n_proc; rank++)
19839 {
19840 // Set the offset for the current processor
19842
19843#ifdef PARANOID
19844 // Check that total number of elements processed matches those
19845 // in individual batches
19847#endif
19848
19849 // Counter for number of elements
19850 unsigned el_count = 0;
19851
19852 // How many baches are there for current rank?
19854
19855 // Add to vector of doubles to save on number of comms
19856 send_data.push_back(double(nbatch));
19857
19858 // Loop over batches of elemnts associated with same root
19859 for (unsigned b = 0; b < nbatch; b++)
19860 {
19861 // How many elements are to be sent in this batch?
19862 unsigned nel = nelement_batch_for_processor[rank][b];
19863
19864 // Get the unique number of the root element in unrefined mesh for
19865 // all the elements in this batch
19866 unsigned base_el_no =
19868
19869 // Add unsigneds to send data to minimise number of
19870 // communications
19871 send_data.push_back(double(nel));
19872 send_data.push_back(double(base_el_no));
19873
19874 // Loop over batch of elements
19875 for (unsigned e = 0; e < nel; e++)
19876 {
19877 // Get element
19879
19880 // FE?
19881 FiniteElement* fe_pt = dynamic_cast<FiniteElement*>(el_pt);
19882 if (fe_pt != 0)
19883 {
19884 // Loop over nodes
19885 unsigned nnod = fe_pt->nnode();
19886 for (unsigned j = 0; j < nnod; j++)
19887 {
19888 Node* nod_pt = fe_pt->node_pt(j);
19889
19890 // Reconstruct the nodal values/position from the node's
19891 // possible hanging node representation to be on the safe side
19892 unsigned n_value = nod_pt->nvalue();
19893 unsigned nt = nod_pt->ntstorage();
19894 Vector<double> values(n_value);
19895 unsigned n_dim = nod_pt->ndim();
19896 Vector<double> position(n_dim);
19897
19898 // Loop over all history values
19899 for (unsigned t = 0; t < nt; t++)
19900 {
19901 nod_pt->value(t, values);
19902 for (unsigned i = 0; i < n_value; i++)
19903 {
19904 nod_pt->set_value(t, i, values[i]);
19905 }
19906 nod_pt->position(t, position);
19907 for (unsigned i = 0; i < n_dim; i++)
19908 {
19909 nod_pt->x(t, i) = position[i];
19910 }
19911 }
19912
19913
19914 // Has the node already been done for current rank?
19915 if (!node_done[rank][nod_pt])
19916 {
19917 // Now it has been done
19918 node_done[rank][nod_pt] = true;
19919
19920 // Store number of values (as double) to allow for resizing
19921 // before read (req'd in case we store data that
19922 // got introduced by attaching FaceElements to bulk)
19923 send_data.push_back(double(n_value));
19924
19925 // Check if it's a boundary node
19927 dynamic_cast<BoundaryNodeBase*>(nod_pt);
19928
19929 // Not a boundary node
19930 if (bnod_pt == 0)
19931 {
19932#ifdef PARANOID
19933 // Record status for checking
19934 send_data.push_back(double(0));
19935#endif
19936 }
19937 // Yes it's a boundary node
19938 else
19939 {
19940#ifdef PARANOID
19941 // Record status for checking
19942 send_data.push_back(double(1));
19943#endif
19944 // Get pointer to the map of indices associated with
19945 // additional values created by face elements
19946 std::map<unsigned, unsigned>* map_pt =
19947 bnod_pt->index_of_first_value_assigned_by_face_element_pt();
19948
19949 // No additional values created
19950 if (map_pt == 0)
19951 {
19952 send_data.push_back(double(0));
19953 }
19954 // Created additional values
19955 else
19956 {
19957 // How many?
19958 send_data.push_back(double(map_pt->size()));
19959
19960 // Loop over entries in map and add to send data
19961 for (std::map<unsigned, unsigned>::iterator p =
19962 map_pt->begin();
19963 p != map_pt->end();
19964 p++)
19965 {
19966 send_data.push_back(double((*p).first));
19967 send_data.push_back(double((*p).second));
19968 }
19969 }
19970 }
19971
19972 // Add the actual values
19973 nod_pt->add_values_to_vector(send_data);
19974 }
19975 }
19976 }
19977
19978 // Now add internal data
19980
19981 // Bump up counter in long vector of elements
19982 el_count++;
19983 }
19984 }
19985
19986
19987#ifdef PARANOID
19988 // Check that total number of elements matches the total of those
19989 // in batches
19990 if (total_nel != el_count)
19991 {
19992 std::ostringstream error_message;
19993 error_message
19994 << "total_nel: " << total_nel << " "
19995 << " doesn't match total number of elements sent in batch: "
19996 << el_count << "\n";
19997 throw OomphLibError(error_message.str(),
20000 }
20001#endif
20002
20003 // Find the number of data added to the vector
20005 }
20006 }
20007
20008
20009 //==========================================================================
20010 /// Get flat-packed refinement pattern for each root element in current
20011 /// mesh (labeled by unique number of root element in unrefined base mesh).
20012 /// The vector stored for each root element contains the following
20013 /// information:
20014 /// - First entry: Number of tree nodes (not just leaves!) in refinement
20015 /// tree emanating from this root [Zero if root element is not refineable]
20016 /// - Loop over all refinement levels
20017 /// - Loop over all tree nodes (not just leaves!)
20018 /// - If associated element exists when the mesh has been refined to
20019 /// this level (either because it has been refined to this level or
20020 /// because it's less refined): 1
20021 /// - If the element is to be refined: 1; else: 0
20022 /// - else (element doesn't exist when mesh is refined to this level
20023 /// (because it's more refined): 0
20024 /// .
20025 /// .
20026 /// .
20027 //==========================================================================
20031 const unsigned& max_refinement_level_overall,
20033 {
20034 // Map to store whether the root element has been visited yet
20035 std::map<RefineableElement*, bool> root_el_done;
20036
20037 // Get the number of submeshs, if there are no submeshes, then
20038 // increase the counter so that the loop below also work for the only
20039 // one mesh in the problem
20040 unsigned n_mesh = nsub_mesh();
20041 if (n_mesh == 0)
20042 {
20043 n_mesh = 1;
20044 }
20045 // Go for the nonhalo elements only in the TreeBaseMeshes
20046 for (unsigned i_mesh = 0; i_mesh < n_mesh; i_mesh++)
20047 {
20048 // Only work with structured
20050 dynamic_cast<TriangleMeshBase*>(mesh_pt(i_mesh));
20051 if (!(sub_mesh_pt != 0))
20052 {
20053 const unsigned nele_submesh = mesh_pt(i_mesh)->nelement();
20054 for (unsigned e = 0; e < nele_submesh; e++)
20055 {
20056 // Get pointer to element
20058
20059 // Ignore halos
20060 if (!el_pt->is_halo())
20061 {
20062 // Is it refineable? No!
20064 dynamic_cast<RefineableElement*>(el_pt);
20065 if (ref_el_pt == 0)
20066 {
20067 // The element is not refineable - stick a zero in refinement_info
20068 // indicating that there are no tree nodes following
20070#ifdef PARANOID
20071 if (e == 0)
20072 {
20073 throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
20076 }
20077#endif
20078 e -= 1;
20080 }
20081 // Refineable
20082 else
20083 {
20084 // Get the root element
20085 RefineableElement* root_el_pt = ref_el_pt->root_element_pt();
20086
20087 // Has this root been visited yet?
20089 {
20090 // Get unique number of root element in base mesh
20091 unsigned root_element_number =
20093
20094#ifdef PARANOID
20095 if (root_element_number == 0)
20096 {
20097 throw OomphLibError(
20098 "Base_mesh_element_number_plus_one[...]=0",
20101 }
20102#endif
20104
20105 // Get all the nodes associated with this root element
20107 root_el_pt->tree_pt()->stick_all_tree_nodes_into_vector(
20109
20110 // How many tree nodes are there?
20111 unsigned n_tree_nodes = all_tree_nodes_pt.size();
20113 .push_back(n_tree_nodes);
20114
20115 // Loop over all levels
20116 for (unsigned current_level = 0;
20118 current_level++)
20119 {
20120 // Loop over all tree nodes
20121 for (unsigned e = 0; e < n_tree_nodes; e++)
20122 {
20123 // What's the level of this tree node?
20124 unsigned level = all_tree_nodes_pt[e]->level();
20125
20126 // Element exists at this refinement level of the mesh
20127 // if it's at this level or it's at a lower level and a leaf
20128 if ((level == current_level) ||
20129 ((level < current_level) &&
20130 (all_tree_nodes_pt[e]->is_leaf())))
20131 {
20133 .push_back(1);
20134
20135 // If it's at this level, and not a leaf, then it will
20136 // need to be refined in the new mesh
20137 if ((level == current_level) &&
20138 (!all_tree_nodes_pt[e]->is_leaf()))
20139 {
20142 .push_back(1);
20143 }
20144 // Element exists at this level and is a leaf so it
20145 // doesn't have to be refined
20146 else
20147 {
20150 .push_back(0);
20151 }
20152 }
20153 // Element does not exist at this level so it doesn't have
20154 // to be refined
20155 else
20156 {
20158 .push_back(0);
20159 }
20160 }
20161 }
20162 // Now we've done it
20163 root_el_done[root_el_pt] = true;
20164 }
20165 }
20166
20167 } // if (!el_pt->is_halo())
20168 } // for (e < nele_submesh)
20169 } // if (!(sub_mesh_pt!=0))
20170 } // for (i_mesh < n_mesh)
20171 }
20172
20173 //==========================================================================
20174 /// Load balance helper routine: Function performs max_level_overall
20175 /// successive refinements of the problem's mesh(es) using the following
20176 /// procdure: Given ID of root element, root_element_id, and current
20177 /// refinement level, level, the e-th entry in
20178 /// refinement_info_for_root_elements[root_element_id][level][e] is equal
20179 /// to 2 if the e-th element (using the enumeration when the mesh has been
20180 /// refined to the level-th level) is to be refined during the next
20181 /// refinement; it's 1 if it's not to be refined.
20182 //==========================================================================
20185 const unsigned& max_level_overall)
20186 {
20187 // Loop over sub meshes
20188 unsigned n_sub_mesh = nsub_mesh();
20189 unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20190 for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20191 {
20192 // Choose the right mesh
20193 Mesh* my_mesh_pt = 0;
20194 if (n_sub_mesh == 0)
20195 {
20196 my_mesh_pt = mesh_pt();
20197 }
20198 else
20199 {
20201 }
20202
20203 // Number of elements on this processor -- currently all elements
20204 // are "base" elements since the mesh hasn't been refined.
20205 unsigned n_el_on_this_proc = my_mesh_pt->nelement();
20206
20207 // Storage for actual refinement pattern:
20208 // to_be_refined_on_this_proc[level][e] contains the element number
20209 // of the e-th element that is to refined at the level-th refinement level
20211
20212 // Count, at each level, the total number of elements in the mesh
20213 // (we can accumulate this because we know that elements are
20214 // enumerated tree by tree).
20216
20217 // Loop over levels where refinement is taking place
20218 for (unsigned level = 0; level < max_level_overall; level++)
20219 {
20220 // Loop over roots = unrefined elements on this processor in order.
20221 // Note that this loops over the trees in unique order
20222 for (unsigned e = 0; e < n_el_on_this_proc; e++)
20223 {
20224 // Get the (root) element
20225 FiniteElement* el_pt = my_mesh_pt->finite_element_pt(e);
20226
20227 // What is its unique number in the base mesh
20229#ifdef PARANOID
20230 if (root_el_no == 0)
20231 {
20232 throw OomphLibError("Base_mesh_element_number_plus_one[...]=0",
20235 }
20236#endif
20237 root_el_no -= 1;
20238
20239 // Number of refinements to be performed starting from current
20240 // root element
20241 unsigned n_refinements =
20243
20244 // Perform refinement?
20245 if (level < n_refinements)
20246 {
20247 // Loop over elements at this level
20248 unsigned n_el =
20250 for (unsigned ee = 0; ee < n_el; ee++)
20251 {
20252 // Refinement code 2: Element is to be refined at this
20253 // level
20255 {
20256 to_be_refined_on_this_proc[level].push_back(
20257 el_count_on_this_proc[level]);
20258 el_count_on_this_proc[level]++;
20259 }
20260 // Refinement code 1: Element should not be refined at this
20261 // level -- keep going
20263 [ee] == 1)
20264 {
20265 el_count_on_this_proc[level]++;
20266 }
20267 }
20268 }
20269
20270 } // end of loop over elements on proc; all of which should be root
20271 }
20272
20273 // Now do the actual refinement
20276 if (ref_mesh_pt != 0)
20277 {
20278 ref_mesh_pt->refine_base_mesh(to_be_refined_on_this_proc);
20279 }
20280 }
20281
20282 // Rebuild global mesh after refinement
20283 if (n_sub_mesh != 0)
20284 {
20285 // Rebuild the global mesh
20287 }
20288 }
20289
20290
20291 //====================================================================
20292 /// Helper function to re-setup the Base_mesh enumeration
20293 /// (used during load balancing) after pruning.
20294 //====================================================================
20296 {
20297 // Storage for number of processors and current processor
20298 int n_proc = this->communicator_pt()->nproc();
20299 int my_rank = this->communicator_pt()->my_rank();
20300
20301 // Loop over sub meshes
20302 unsigned n_sub_mesh = nsub_mesh();
20303 unsigned max_mesh = std::max(n_sub_mesh, unsigned(1));
20304 for (unsigned i_mesh = 0; i_mesh < max_mesh; i_mesh++)
20305 {
20306 // Choose the right mesh
20307 Mesh* my_mesh_pt = 0;
20308 if (n_sub_mesh == 0)
20309 {
20310 my_mesh_pt = mesh_pt();
20311 }
20312 else
20313 {
20315 }
20316
20317 // Only work with structured meshes
20319 dynamic_cast<TriangleMeshBase*>(my_mesh_pt);
20320 if (!(sub_mesh_pt != 0))
20321 {
20322 // Storage for number of data to be sent to each processor
20324
20325 // Storage for all values to be sent to all processors
20327
20328 // Start location within send_data for data to be sent to each processor
20330
20331 // Loop over all processors
20332 for (int rank = 0; rank < n_proc; rank++)
20333 {
20334 // Set the offset for the current processor
20336
20337 // Don't bother to do anything if the processor in the loop is the
20338 // current processor
20339 if (rank != my_rank)
20340 {
20341 // Get root haloed elements with that processor
20343 my_mesh_pt->root_haloed_element_pt(rank);
20344 unsigned nel = root_haloed_elements_pt.size();
20345
20346 // Store element numbers for send
20347 for (unsigned e = 0; e < nel; e++)
20348 {
20351 }
20352 }
20353
20354 // Find the number of data added to the vector
20356 }
20357
20358 // Storage for the number of data to be received from each processor
20360
20361 // Now send numbers of data to be sent between all processors
20362 MPI_Alltoall(&send_n[0],
20363 1,
20364 MPI_INT,
20365 &receive_n[0],
20366 1,
20367 MPI_INT,
20368 this->communicator_pt()->mpi_comm());
20369
20370 // We now prepare the data to be received
20371 // by working out the displacements from the received data
20373 int receive_data_count = 0;
20374 for (int rank = 0; rank < n_proc; ++rank)
20375 {
20376 // Displacement is number of data received so far
20379 }
20380
20381 // Now resize the receive buffer for all data from all processors
20382 // Make sure that it has a size of at least one
20383 if (receive_data_count == 0)
20384 {
20386 }
20388
20389 // Make sure that the send buffer has size at least one
20390 // so that we don't get a segmentation fault
20391 if (send_data.size() == 0)
20392 {
20393 send_data.resize(1);
20394 }
20395
20396 // Now send the data between all the processors
20398 &send_n[0],
20401 &receive_data[0],
20402 &receive_n[0],
20405 this->communicator_pt()->mpi_comm());
20406
20407 // Now use the received data to update the halo element numbers in
20408 // base mesh
20409 for (int send_rank = 0; send_rank < n_proc; send_rank++)
20410 {
20411 // Don't bother to do anything for the processor corresponding to the
20412 // current processor or if no data were received from this processor
20413 if ((send_rank != my_rank) && (receive_n[send_rank] != 0))
20414 {
20415 // Counter for the data within the large array
20417
20418 // Get root halo elements with that processor
20420 my_mesh_pt->root_halo_element_pt(send_rank);
20421 unsigned nel = root_halo_elements_pt.size();
20422
20423 // Read in element numbers
20424 for (unsigned e = 0; e < nel; e++)
20425 {
20430 }
20431 }
20432
20433 } // End of data is received
20434
20435 } // if (!(sub_mesh_pt!=0))
20436
20437 } // for (i_mesh<max_mesh)
20438 }
20439
20440#endif
20441
20442 /// Instantiation of public flag to allow suppression of warning
20443 /// messages re reading in unstructured meshes during restart.
20445 false;
20446
20447
20448} // namespace oomph
e
Definition cfortran.h:571
cstr elem_len * i
Definition cfortran.h:603
char t
Definition cfortran.h:568
A class that is used to define the functions used to assemble the elemental contributions to the resi...
virtual unsigned ndof(GeneralisedElement *const &elem_pt)
Return the number of degrees of freedom in the element elem_pt.
virtual void synchronise()
Function that is used to perform any synchronisation required during the solution.
virtual int bifurcation_type() const
Return an unsigned integer to indicate whether the handler is a bifurcation tracking handler....
virtual void get_hessian_vector_products(GeneralisedElement *const &elem_pt, Vector< double > const &Y, DenseMatrix< double > const &C, DenseMatrix< double > &product)
Calculate the product of the Hessian (derivative of Jacobian with respect to all variables) an eigenv...
virtual double * bifurcation_parameter_pt() const
Return a pointer to the bifurcation parameter in bifurcation tracking problems.
virtual void get_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction(s) associated with the bifurcation that has been detected in bifurcation tra...
virtual void get_residuals(GeneralisedElement *const &elem_pt, Vector< double > &residuals)
Return the contribution to the residuals of the element elem_pt.
virtual unsigned long eqn_number(GeneralisedElement *const &elem_pt, const unsigned &ieqn_local)
Return the global equation number of the local unknown ieqn_local in elem_pt.
virtual void get_all_vectors_and_matrices(GeneralisedElement *const &elem_pt, Vector< Vector< double > > &vec, Vector< DenseMatrix< double > > &matrix)
Calculate all desired vectors and matrices provided by the element elem_pt.
virtual void get_jacobian(GeneralisedElement *const &elem_pt, Vector< double > &residuals, DenseMatrix< double > &jacobian)
Calculate the elemental Jacobian matrix "d equation / d variable" for elem_pt.
A custom linear solver class that is used to solve a block-factorised version of the Hopf bifurcation...
A class that contains the information required by Nodes that are located on Mesh boundaries....
Definition nodes.h:1996
//////////////////////////////////////////////////////////////// ////////////////////////////////////...
Definition matrices.h:2791
void build_without_copy(T *value, int *row_index, int *column_start, const unsigned long &nnz, const unsigned long &n, const unsigned long &m)
Function to build matrix from pointers to arrays which hold the column starts, row indices and non-ze...
Definition matrices.h:3199
A class for compressed row matrices. This is a distributable object.
Definition matrices.h:888
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the matrix are redistributed to match the new distribution. In a non-MPI build this m...
Definition matrices.cc:2575
void build_without_copy(const unsigned &ncol, const unsigned &nnz, double *value, int *column_index, int *row_start)
keeps the existing distribution and just matrix that is stored without copying the matrix data
Definition matrices.cc:1710
void build(const LinearAlgebraDistribution *distribution_pt, const unsigned &ncol, const Vector< double > &value, const Vector< int > &column_index, const Vector< int > &row_start)
build method: vector of values, vector of column indices, vector of row starts and number of rows and...
Definition matrices.cc:1672
A Base class for DGElements.
A class that represents a collection of data; each Data object may contain many different individual ...
Definition nodes.h:86
void copy(Data *orig_data_pt)
Copy Data values from specified Data object.
Definition nodes.cc:601
void set_value(const unsigned &i, const double &value_)
Set the i-th stored data value to specified value. The only reason that we require an explicit set fu...
Definition nodes.h:271
unsigned nvalue() const
Return number of values stored in data object (incl pinned ones).
Definition nodes.h:483
double value(const unsigned &i) const
Return i-th stored value. This function is not virtual so that it can be inlined. This means that if ...
Definition nodes.h:293
long & eqn_number(const unsigned &i)
Return the equation number of the i-th stored variable.
Definition nodes.h:367
Class of matrices containing doubles, and stored as a DenseMatrix<double>, but with solving functiona...
Definition matrices.h:1271
void initialise(const T &val)
Initialize all values in the matrix to val.
Definition matrices.h:514
void resize(const unsigned long &n)
Resize to a square nxn matrix; any values already present will be transfered.
Definition matrices.h:498
bool distribution_built() const
if the communicator_pt is null then the distribution is not setup then false is returned,...
LinearAlgebraDistribution * distribution_pt() const
access to the LinearAlgebraDistribution
Information for documentation of results: Directory and file number to enable output in the form RESL...
std::string & label()
String used (e.g.) for labeling output files.
bool is_doc_enabled() const
Are we documenting?
void disable_doc()
Disable documentation.
std::string directory() const
Output directory.
unsigned & number()
Number used (e.g.) for labeling output files.
A class that stores the halo/haloed entries required when using a DoubleVectorWithHaloEntries....
void setup_halo_dofs(const std::map< unsigned, double * > &halo_data_pt, Vector< double * > &halo_dof_pt)
Function that sets up a vector of pointers to halo data, index using the scheme in Local_index.
===================================================================== An extension of DoubleVector th...
void build_halo_scheme(DoubleVectorHaloScheme *const &halo_scheme_pt)
Construct the halo scheme and storage for the halo data.
void sum_all_halo_and_haloed_values()
Sum all the data, store in the master (haloed) data and then synchronise.
double & global_value(const unsigned &i)
Direct access to global entry.
A vector in the mathematical sense, initially developed for linear algebra type applications....
double max() const
returns the maximum coefficient
void build(const DoubleVector &old_vector)
Just copys the argument DoubleVector.
void redistribute(const LinearAlgebraDistribution *const &dist_pt)
The contents of the vector are redistributed to match the new distribution. In a non-MPI rebuild this...
double * values_pt()
access function to the underlying values
void clear()
wipes the DoubleVector
A class that is used to define the functions used to assemble the elemental contributions to the mass...
virtual void solve_eigenproblem(Problem *const &problem_pt, const int &n_eval, Vector< std::complex< double > > &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &do_adjoint_problem=false)
Solve the real eigenproblem that is assembled by elements in a mesh in a Problem object....
virtual void solve_eigenproblem_legacy(Problem *const &problem_pt, const int &n_eval, Vector< std::complex< double > > &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &do_adjoint_problem=false)=0
Eigensolver. This takes a pointer to a problem and returns a vector of complex numbers representing t...
Base class for spatial error estimators.
A class that is used to define the functions used to assemble and invert the mass matrix when taking ...
A Base class for explicit timesteppers.
virtual void timestep(ExplicitTimeSteppableObject *const &object_pt, const double &dt)=0
Pure virtual function that is used to advance time in the object.
FaceElements are elements that coincide with the faces of higher-dimensional "bulk" elements....
Definition elements.h:4338
A general Finite Element class.
Definition elements.h:1313
void position(const Vector< double > &zeta, Vector< double > &r) const
Return the parametrised position of the FiniteElement in its incarnation as a GeomObject,...
Definition elements.h:2676
double size() const
Calculate the size of the element (length, area, volume,...) in Eulerian computational coordinates....
Definition elements.cc:4290
unsigned nnode() const
Return the number of nodes.
Definition elements.h:2210
Node *& node_pt(const unsigned &n)
Return a pointer to the local node n.
Definition elements.h:2175
virtual void describe_local_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the local dofs of the element[s]. The ostream specifies the output stream to whi...
Definition elements.cc:1709
A Generalised Element class.
Definition elements.h:73
bool is_halo() const
Is this element a halo?
Definition elements.h:1163
void read_internal_eqn_numbers_from_vector(const Vector< long > &vector_of_eqn_numbers, unsigned &index)
Read all equation numbers associated with internal data from the vector starting from index....
Definition elements.cc:670
unsigned ndof() const
Return the number of equations/dofs in the element.
Definition elements.h:835
unsigned long eqn_number(const unsigned &ieqn_local) const
Return the global equation number corresponding to the ieqn_local-th local equation number.
Definition elements.h:704
Data *& internal_data_pt(const unsigned &i)
Return a pointer to i-th internal data object.
Definition elements.h:622
void add_internal_data_values_to_vector(Vector< double > &vector_of_values)
Add all internal data and time history values to the vector in the internal storage order.
Definition elements.cc:628
unsigned ninternal_data() const
Return the number of internal data objects.
Definition elements.h:823
void add_internal_eqn_numbers_to_vector(Vector< long > &vector_of_eqn_numbers)
Add all equation numbers associated with internal data to the vector in the internal storage order.
Definition elements.cc:655
virtual void assign_local_eqn_numbers(const bool &store_local_dof_pt)
Setup the arrays of local equation numbers for the element. If the optional boolean argument is true,...
Definition elements.cc:691
void describe_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the dofs of the element. The ostream specifies the output stream to which the de...
Definition elements.cc:551
virtual void complete_setup_of_dependencies()
Complete the setup of any additional dependencies that the element may have. Empty virtual function t...
Definition elements.h:974
void read_internal_data_values_from_vector(const Vector< double > &vector_of_values, unsigned &index)
Read all internal data and time history values from the vector starting from index....
Definition elements.cc:642
unsigned ndim() const
Access function to # of Eulerian coordinates.
TimeStepper *& time_stepper_pt()
Access function for pointer to time stepper: Null if object is not time-dependent.
Class that contains data for hanging nodes.
Definition nodes.h:742
Node *const & master_node_pt(const unsigned &i) const
Return a pointer to the i-th master node.
Definition nodes.h:791
unsigned nmaster() const
Return the number of master nodes.
Definition nodes.h:785
void set_master_node_pt(const unsigned &i, Node *const &master_node_pt, const double &weight)
Set the pointer to the i-th master node and its weight.
Definition nodes.cc:1474
A class that is used to assemble the augmented system that defines a Hopf bifurcation....
Class for the LAPACK QZ eigensolver.
Describes the distribution of a distributable linear algebra type object. Typically this is a contain...
bool distributed() const
access function to the distributed - indicates whether the distribution is serial or distributed
OomphCommunicator * communicator_pt() const
const access to the communicator pointer
void build(const OomphCommunicator *const comm_pt, const unsigned &first_row, const unsigned &nrow_local, const unsigned &nrow=0)
Sets the distribution. Takes first_row, nrow_local and nrow as arguments. If nrow is not provided or ...
unsigned nrow() const
access function to the number of global rows.
unsigned nrow_local() const
access function for the num of local rows on this processor. If no MPI then Nrow is returned.
virtual void solve(Problem *const &problem_pt, DoubleVector &result)=0
Solver: Takes pointer to problem and returns the results vector which contains the solution of the li...
virtual void enable_resolve()
Enable resolve (i.e. store matrix and/or LU decomposition, say) Virtual so it can be overloaded to pe...
virtual void enable_computation_of_gradient()
function to enable the computation of the gradient required for the globally convergent Newton method
virtual void resolve(const DoubleVector &rhs, DoubleVector &result)
Resolve the system defined by the last assembled jacobian and the rhs vector. Solution is returned in...
void get_gradient(DoubleVector &gradient)
function to access the gradient, provided it has been computed
void reset_gradient()
function to reset the size of the gradient before each Newton solve
virtual void disable_resolve()
Disable resolve (i.e. store matrix and/or LU decomposition, say) This function simply resets an inter...
bool is_resolve_enabled() const
Boolean flag indicating if resolves are enabled.
static bool mpi_has_been_initialised()
return true if MPI has been initialised
static OomphCommunicator * communicator_pt()
access to global communicator. This is the oomph-lib equivalent of MPI_COMM_WORLD
A general mesh class.
Definition mesh.h:67
bool does_pointer_correspond_to_mesh_data(double *const &parameter_pt)
Does the double pointer correspond to any mesh data.
Definition mesh.cc:2471
void remove_boundary_node(const unsigned &b, Node *const &node_pt)
Remove a node from the boundary b.
Definition mesh.cc:221
GeneralisedElement *& external_halo_element_pt(const unsigned &p, const unsigned &e)
Access fct to the e-th external halo element in this Mesh whose non-halo counterpart is held on proce...
Definition mesh.h:2251
void flush_element_and_node_storage()
Flush storage for elements and nodes by emptying the vectors that store the pointers to them....
Definition mesh.h:407
FiniteElement * finite_element_pt(const unsigned &e) const
Upcast (downcast?) to FiniteElement (needed to access FiniteElement member functions).
Definition mesh.h:473
void check_halo_schemes(DocInfo &doc_info, double &max_permitted_error_for_halo_check)
Check halo and shared schemes on the mesh.
Definition mesh.cc:6881
virtual void set_mesh_level_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Function that can be used to set any additional timestepper data stored at the Mesh (as opposed to no...
Definition mesh.cc:2402
void describe_local_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the local dofs of the elements. The ostream specifies the output stream to which...
Definition mesh.cc:746
Node *& node_pt(const unsigned long &n)
Return pointer to global node n.
Definition mesh.h:436
void set_nodal_and_elemental_time_stepper(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data)
Set the timestepper associated with all nodal and elemental data stored in the mesh.
Definition mesh.h:1032
void shift_time_values()
Shift time-dependent data along for next timestep: Deal with nodal Data/positions and the element's i...
Definition mesh.cc:2326
void prune_halo_elements_and_nodes(Vector< GeneralisedElement * > &deleted_element_pt, const bool &report_stats=false)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition mesh.h:1667
void calculate_predictions()
Calculate predictions for all Data and positions associated with the mesh, usually used in adaptive t...
Definition mesh.cc:2366
void describe_dofs(std::ostream &out, const std::string &current_string) const
Function to describe the dofs of the Mesh. The ostream specifies the output stream to which the descr...
Definition mesh.cc:711
unsigned long nnode() const
Return number of nodes in the mesh.
Definition mesh.h:596
GeneralisedElement *& element_pt(const unsigned long &e)
Return pointer to element e.
Definition mesh.h:448
void delete_all_external_storage()
Wipe the storage for all externally-based elements.
Definition mesh.cc:9190
unsigned nnon_halo_element()
Total number of non-halo elements in this mesh (Costly call computes result on the fly)
Definition mesh.h:1817
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get all the halo data stored in the mesh and add pointers to the data to the map, indexed by global e...
Definition mesh.cc:4749
void assign_initial_values_impulsive()
Assign initial values for an impulsive start.
Definition mesh.cc:2288
void assign_local_eqn_numbers(const bool &store_local_dof_pt)
Assign the local equation numbers in all elements If the boolean argument is true then also store poi...
Definition mesh.cc:765
virtual void read(std::ifstream &restart_file)
Read solution from restart file.
Definition mesh.cc:1130
void set_consistent_pinned_values_for_continuation(ContinuationStorageScheme *const &continuation_stepper_pt)
Set consistent values for pinned data in continuation.
Definition mesh.cc:2436
unsigned long assign_global_eqn_numbers(Vector< double * > &Dof_pt)
Assign the global equation numbers in the Data stored at the nodes and also internal element Data....
Definition mesh.cc:677
virtual void distribute(OomphCommunicator *comm_pt, const Vector< unsigned > &element_domain, Vector< GeneralisedElement * > &deleted_element_pt, DocInfo &doc_info, const bool &report_stats, const bool &overrule_keep_as_halo_element_status)
Distribute the problem and doc; make this virtual to allow overloading for particular meshes where fu...
Definition mesh.cc:4959
void null_external_halo_node(const unsigned &p, Node *nod_pt)
Null out specified external halo node (used when deleting duplicates)
Definition mesh.cc:8569
unsigned long nelement() const
Return number of elements in the mesh.
Definition mesh.h:590
void merge_meshes(const Vector< Mesh * > &sub_mesh_pt)
Merge meshes. Note: This simply merges the meshes' elements and nodes (ignoring duplicates; no bounda...
Definition mesh.cc:65
unsigned nexternal_halo_element()
Total number of external halo elements in this Mesh.
Definition mesh.h:2222
virtual void dump(std::ofstream &dump_file, const bool &use_old_ordering=true) const
Dump the data in the mesh into a file for restart.
Definition mesh.cc:1088
A class to handle errors in the Newton solver.
Definition problem.h:3081
Nodes are derived from Data, but, in addition, have a definite (Eulerian) position in a space of a gi...
Definition nodes.h:906
void copy(Node *orig_node_pt)
Copy all nodal data from specified Node object.
Definition nodes.cc:1916
unsigned ndim() const
Return (Eulerian) spatial dimension of the node.
Definition nodes.h:1054
double & x(const unsigned &i)
Return the i-th nodal coordinate.
Definition nodes.h:1060
bool is_hanging() const
Test whether the node is geometrically hanging.
Definition nodes.h:1285
double value(const unsigned &i) const
Return i-th value (dofs or pinned) at this node either directly or via hanging node representation....
Definition nodes.cc:2408
HangInfo *const & hanging_pt() const
Return pointer to hanging node data (this refers to the geometric hanging node status) (const version...
Definition nodes.h:1228
An oomph-lib wrapper to the MPI_Comm communicator object. Just contains an MPI_Comm object (which is ...
std::ostream *& stream_pt()
Access function for the stream pointer.
An OomphLibError object which should be thrown when an run-time error is encountered....
An OomphLibWarning object which should be created as a temporary object to issue a warning....
A class that is used to assemble the residuals in parallel by overloading the get_all_vectors_and_mat...
A class that is used to define the functions used when assembling the derivatives of the residuals wi...
////////////////////////////////////////////////////////////////// //////////////////////////////////...
Definition problem.h:151
virtual void actions_after_implicit_timestep()
Actions that should be performed after each implicit time step. This is needed when one wants to solv...
Definition problem.h:1070
bool Always_take_one_newton_step
Boolean to indicate whether a Newton step should be taken even if the initial residuals are below the...
Definition problem.h:2326
bool Jacobian_reuse_is_enabled
Is re-use of Jacobian in Newton iteration enabled? Default: false.
Definition problem.h:618
virtual void actions_after_newton_solve()
Any actions that are to be performed after a complete Newton solve, e.g. post processing....
Definition problem.h:1038
void parallel_sparse_assemble(const LinearAlgebraDistribution *const &dist_pt, Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residuals)
Helper method to assemble CRDoubleMatrices from distributed on multiple processors.
Definition problem.cc:6533
void describe_dofs(std::ostream &out= *(oomph_info.stream_pt())) const
Function to describe the dofs in terms of the global equation number, i.e. what type of value (nodal ...
Definition problem.cc:2445
virtual void sparse_assemble_row_or_column_compressed(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Protected helper function that is used to assemble the Jacobian matrix in the case when the storage i...
Definition problem.cc:4461
double * global_dof_pt(const unsigned &i)
Return a pointer to the dof, indexed by global equation number which may be haloed or stored locally....
Definition problem.h:1768
bool Store_local_dof_pt_in_elements
Boolean to indicate whether local dof pointers should be stored in the elements.
Definition problem.h:226
virtual void actions_before_newton_step()
Any actions that are to be performed before each individual Newton step. Most likely to be used for d...
Definition problem.h:1053
void remove_duplicate_data(Mesh *const &mesh_pt, bool &actually_removed_some_data)
Private helper function to remove repeated data in external haloed elements in specified mesh....
Definition problem.cc:2654
bool Bifurcation_detection
Boolean to control bifurcation detection via determinant of Jacobian.
Definition problem.h:790
void get_flat_packed_refinement_pattern_for_load_balancing(const Vector< unsigned > &old_domain_for_base_element, const Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned > > &flat_packed_refinement_info_for_root)
Get flat-packed refinement pattern for each root element in current mesh (labeled by unique number of...
Definition problem.cc:20028
void adapt()
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error e...
Definition problem.h:2988
virtual void actions_before_newton_solve()
Any actions that are to be performed before a complete Newton solve (e.g. adjust boundary conditions)...
Definition problem.h:1032
Vector< unsigned > First_el_for_assembly
First element to be assembled by given processor for non-distributed problem (only kept up to date wh...
Definition problem.h:519
unsigned long assign_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Assign all equation numbers for problem: Deals with global data (= data that isn't attached to any el...
Definition problem.cc:2075
void refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund refinement of (all) refineable (sub)mesh(es) uniformly as many times as...
Definition problem.cc:15638
void build_global_mesh()
Build the global mesh by combining the all the submeshes. Note: The nodes boundary information refers...
Definition problem.cc:1579
unsigned unrefine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem. Return 0 for success,...
Definition problem.cc:16024
void assign_initial_values_impulsive()
Initialise data and nodal positions to simulate impulsive start from initial configuration/solution.
Definition problem.cc:11692
double Theta_squared
Value of the scaling parameter required so that the parameter occupies the desired proportion of the ...
Definition problem.h:742
virtual void get_eigenproblem_matrices(CRDoubleMatrix &mass_matrix, CRDoubleMatrix &main_matrix, const double &shift=0.0)
Get the matrices required by a eigensolver. If the shift parameter is non-zero the second matrix will...
Definition problem.cc:8602
Vector< double > Dof_derivative
Storage for the derivative of the problem variables wrt arc-length.
Definition problem.h:774
double & dof_current(const unsigned &i)
Access function to the current value of the i-th (local) dof at the start of a continuation step.
Definition problem.h:1182
virtual void sparse_assemble_row_or_column_compressed_with_lists(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition problem.cc:4907
virtual void sparse_assemble_row_or_column_compressed_with_two_arrays(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition problem.cc:6038
friend class BlockHopfLinearSolver
Definition problem.h:162
void synchronise_dofs(const bool &do_halos, const bool &do_external_halos)
Synchronise the degrees of freedom by overwriting the haloed values with their non-halo counterparts ...
Definition problem.cc:16655
virtual void actions_before_distribute()
Actions to be performed before a (mesh) distribution.
Definition problem.h:1116
friend class FoldHandler
Definition problem.h:153
Distributed_problem_matrix_distribution Dist_problem_matrix_distribution
The distributed matrix distribution method 1 - Automatic - the Problem distribution is employed,...
Definition problem.h:956
DoubleVectorHaloScheme * Halo_scheme_pt
Pointer to the halo scheme for any global vectors that have the Dof_distribution.
Definition problem.h:577
virtual void actions_after_change_in_global_parameter(double *const &parameter_pt)
Actions that are to be performed when the global parameter addressed by parameter_pt has been changed...
Definition problem.h:1133
void p_refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
p-refine (one and only!) mesh by refining the elements identified by their numbers relative to the pr...
Definition problem.cc:15352
void setup_dof_halo_scheme()
Function that is used to setup the halo scheme.
Definition problem.cc:386
unsigned long ndof() const
Return the number of dofs.
Definition problem.h:1678
void p_unrefine_uniformly(DocInfo &doc_info)
p-unrefine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem.
Definition problem.cc:16153
virtual void build_mesh()
Function to build the Problem's base mesh; this must be supplied by the user if they wish to use the ...
Definition problem.h:1365
static bool Suppress_warning_about_actions_before_read_unstructured_meshes
Flag to allow suppression of warning messages re reading in unstructured meshes during restart.
Definition problem.h:317
OomphCommunicator * communicator_pt()
access function to the oomph-lib communicator
Definition problem.h:1246
bool Use_default_partition_in_load_balance
Flag to use "default partition" during load balance. Should only be set to true when run in validatio...
Definition problem.h:514
void bifurcation_adapt_helper(unsigned &n_refined, unsigned &n_unrefined, const unsigned &bifurcation_type, const bool &actually_adapt=true)
A function that is used to adapt a bifurcation-tracking problem, which requires separate interpolatio...
Definition problem.cc:13547
void adapt_based_on_error_estimates(unsigned &n_refined, unsigned &n_unrefined, Vector< Vector< double > > &elemental_error)
Adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on the error estimat...
Definition problem.cc:14721
friend class AugmentedBlockFoldLinearSolver
Definition problem.h:160
Vector< double > Elemental_assembly_time
Storage for assembly times (used for load balancing)
Definition problem.h:573
double & dof(const unsigned &i)
i-th dof in the problem
Definition problem.h:1817
bool Jacobian_has_been_computed
Has a Jacobian been computed (and can therefore be re-used if required)? Default: false.
Definition problem.h:622
bool Bypass_increase_in_dof_check_during_pruning
Boolean to bypass check of increase in dofs during pruning.
Definition problem.h:967
virtual void get_dvaluesdt(DoubleVector &f)
Get the time derivative of all values (using get_inverse_mass_matrix_times_residuals(....
Definition problem.cc:3770
void initialise_dt(const double &dt)
Set all timesteps to the same value, dt, and assign weights for all timesteppers in the problem.
Definition problem.cc:13424
void add_to_dofs(const double &lambda, const DoubleVector &increment_dofs)
Add lambda x incremenet_dofs[l] to the l-th dof.
Definition problem.cc:3650
double Timestep_reduction_factor_after_nonconvergence
What it says: If temporally adaptive Newton solver fails to to converge, reduce timestep by this fact...
Definition problem.h:2331
Vector< double > Dof_current
Storage for the present values of the variables.
Definition problem.h:777
double Desired_proportion_of_arc_length
Proportion of the arc-length to taken by the parameter.
Definition problem.h:734
virtual void actions_after_implicit_timestep_and_error_estimation()
Actions that should be performed after each implicit time step. This is needed if your actions_after_...
Definition problem.h:1075
void disable_mass_matrix_reuse()
Turn off recyling of the mass matrix in explicit timestepping schemes.
Definition problem.cc:12025
EigenSolver * Default_eigen_solver_pt
Pointer to the default eigensolver.
Definition problem.h:191
unsigned Nnewton_iter_taken
Actual number of Newton iterations taken during the most recent iteration.
Definition problem.h:603
double * bifurcation_parameter_pt() const
Return pointer to the parameter that is used in the bifurcation detection. If we are not tracking a b...
Definition problem.cc:10274
void copy_haloed_eqn_numbers_helper(const bool &do_halos, const bool &do_external_halos)
A private helper function to copy the haloed equation numbers into the halo equation numbers,...
Definition problem.cc:17144
virtual void sparse_assemble_row_or_column_compressed_with_vectors_of_pairs(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition problem.cc:5322
void send_data_to_be_sent_during_load_balancing(Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement)
Load balance helper routine: Send data to other processors during load balancing.
Definition problem.cc:19142
bool Time_adaptive_newton_crash_on_solve_fail
Bool to specify what to do if a Newton solve fails within a time adaptive solve. Default (false) is t...
Definition problem.h:615
Problem()
Constructor: Allocate space for one time stepper and set all pointers to NULL and set defaults for al...
Definition problem.cc:69
bool is_dparameter_calculated_analytically(double *const &parameter_pt)
Function to determine whether the parameter derivatives are calculated analytically.
Definition problem.h:277
void flush_sub_meshes()
Flush the problem's collection of sub-meshes. Must be followed by call to rebuild_global_mesh().
Definition problem.h:1339
unsigned Sparse_assembly_method
Flag to determine which sparse assembly method to use By default we use assembly by vectors of pairs.
Definition problem.h:641
bool & use_predictor_values_as_initial_guess()
Definition problem.h:2123
void calculate_predictions()
Calculate predictions.
Definition problem.cc:11849
Vector< unsigned > Last_el_plus_one_for_assembly
Last element (plus one) to be assembled by given processor for non-distributed problem (only kept up ...
Definition problem.h:524
virtual void actions_after_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition problem.h:1109
void copy(Problem *orig_problem_pt)
Copy Data values, nodal positions etc from specified problem. Note: This is not a copy constructor....
Definition problem.cc:12058
virtual void get_jacobian(DoubleVector &residuals, DenseDoubleMatrix &jacobian)
Return the fully-assembled Jacobian and residuals for the problem Interface for the case when the Jac...
Definition problem.cc:3921
void set_explicit_time_stepper_pt(ExplicitTimeStepper *const &explicit_time_stepper_pt)
Set the explicit timestepper for the problem. The function will automatically create or resize the Ti...
Definition problem.cc:1672
virtual void actions_after_distribute()
Actions to be performed after a (mesh) distribution.
Definition problem.h:1119
virtual void actions_before_implicit_timestep()
Actions that should be performed before each implicit time step. This is needed when one wants to sol...
Definition problem.h:1064
LinearSolver * Linear_solver_pt
Pointer to the linear solver for the problem.
Definition problem.h:173
bool Doc_time_in_distribute
Protected boolean flag to provide comprehensive timimings during problem distribution....
Definition problem.h:637
unsigned Max_newton_iterations
Maximum number of Newton iterations.
Definition problem.h:599
Vector< Vector< unsigned > > Sparse_assemble_with_arrays_previous_allocation
the number of elements in each row of a compressed matrix in the previous matrix assembly.
Definition problem.h:667
virtual void actions_after_parameter_increase(double *const &parameter_pt)
Empty virtual function; provides hook to perform actions after the increase in the arclength paramete...
Definition problem.h:1161
friend class HopfHandler
Definition problem.h:155
friend class PitchForkHandler
Definition problem.h:154
void calculate_continuation_derivatives(double *const &parameter_pt)
A function to calculate the derivatives wrt the arc-length. This version of the function actually doe...
Definition problem.cc:9908
void store_current_dof_values()
Store the current values of the degrees of freedom.
Definition problem.cc:8787
double & dof_derivative(const unsigned &i)
Access function to the derivative of the i-th (local) dof with respect to the arc length,...
Definition problem.h:1168
bool Problem_has_been_distributed
Has the problem been distributed amongst multiple processors?
Definition problem.h:964
void synchronise_all_dofs()
Perform all required synchronisation in solvers.
Definition problem.cc:16632
void get_all_error_estimates(Vector< Vector< double > > &elemental_error)
Return the error estimates computed by (all) refineable (sub)mesh(es) in the elemental_error structur...
Definition problem.cc:14820
bool Empty_actions_before_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_before_read_unstructured_meshes() function has been called.
Definition problem.h:218
bool Mass_matrix_has_been_computed
Has the mass matrix been computed (and can therefore be reused) Default: false.
Definition problem.h:695
unsigned nglobal_data() const
Return the number of global data values.
Definition problem.h:1690
virtual void actions_before_adapt()
Actions that are to be performed before a mesh adaptation. These might include removing any additiona...
Definition problem.h:1022
void newton_solve()
Use Newton method to solve the problem.
Definition problem.cc:8976
bool First_jacobian_sign_change
Boolean to indicate whether a sign change has occured in the Jacobian.
Definition problem.h:796
void calculate_continuation_derivatives_fd_helper(double *const &parameter_pt)
A function that performs the guts of the continuation derivative calculation in arc-length continuati...
Definition problem.cc:10197
double Continuation_direction
The direction of the change in parameter that will ensure that a branch is followed in one direction ...
Definition problem.h:749
unsigned Parallel_sparse_assemble_previous_allocation
The amount of data allocated during the previous parallel sparse assemble. Used to optimise the next ...
Definition problem.h:960
void enable_mass_matrix_reuse()
Enable recycling of the mass matrix in explicit timestepping schemes. Useful for timestepping on fixe...
Definition problem.cc:12000
void steady_newton_solve(unsigned const &max_adapt=0)
Solve a steady problem using adaptive Newton's method, but in the context of an overall unstady probl...
Definition problem.cc:9485
bool Scale_arc_length
Boolean to control whether arc-length should be scaled.
Definition problem.h:731
double doubly_adaptive_unsteady_newton_solve_helper(const double &dt, const double &epsilon, const unsigned &max_adapt, const unsigned &suppress_resolve_after_spatial_adapt, const bool &first, const bool &shift=true)
Private helper function that actually performs the unsteady "doubly" adaptive Newton solve....
Definition problem.cc:11572
bool Empty_actions_after_read_unstructured_meshes_has_been_called
Boolean to indicate that empty actions_after_read_unstructured_meshes() function has been called.
Definition problem.h:222
virtual void get_residuals(DoubleVector &residuals)
Return the fully-assembled residuals Vector for the problem: Virtual so it can be overloaded in for m...
Definition problem.cc:3800
Vector< GeneralisedElement * > Base_mesh_element_pt
Vector to store the correspondence between a root element and its element number within the global me...
Definition problem.h:548
void get_fd_jacobian(DoubleVector &residuals, DenseMatrix< double > &jacobian)
Return the fully-assembled Jacobian and residuals, generated by finite differences.
Definition problem.cc:7800
virtual void sparse_assemble_row_or_column_compressed_with_two_vectors(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition problem.cc:5675
virtual Problem * make_copy()
Make and return a pointer to the copy of the problem. A virtual function that must be filled in by th...
Definition problem.cc:12204
double Maximum_dt
Maximum desired dt.
Definition problem.h:709
unsigned long set_timestepper_for_all_data(TimeStepper *const &time_stepper_pt, const bool &preserve_existing_data=false)
Set all problem data to have the same timestepper (timestepper_pt) Return the new number of dofs in t...
Definition problem.cc:11765
void activate_pitchfork_tracking(double *const &parameter_pt, const DoubleVector &symmetry_vector, const bool &block_solve=true)
Turn on pitchfork tracking using the augmented system specified in the PitchForkHandler class....
Definition problem.cc:10381
virtual void dump(std::ofstream &dump_file) const
Dump refinement pattern of all refineable meshes and all generic Problem data to file for restart.
Definition problem.cc:12222
bool Use_finite_differences_for_continuation_derivatives
Boolean to specify which scheme to use to calculate the continuation derivatievs.
Definition problem.h:803
bool Arc_length_step_taken
Boolean to indicate whether an arc-length step has been taken.
Definition problem.h:799
void set_pinned_values_to_zero()
Set all pinned values to zero. Used to set boundary conditions to be homogeneous in the copy of the p...
Definition problem.cc:4281
bool Default_set_initial_condition_called
Has default set_initial_condition function been called? Default: false.
Definition problem.h:211
void get_bifurcation_eigenfunction(Vector< DoubleVector > &eigenfunction)
Return the eigenfunction calculated as part of a bifurcation tracking process. If we are not tracking...
Definition problem.cc:10284
double Relaxation_factor
Relaxation fator for Newton method (only a fractional Newton correction is applied if this is less th...
Definition problem.h:592
void add_time_stepper_pt(TimeStepper *const &time_stepper_pt)
Add a timestepper to the problem. The function will automatically create or resize the Time object so...
Definition problem.cc:1631
void refine_selected_elements(const Vector< unsigned > &elements_to_be_refined)
Refine (one and only!) mesh by splitting the elements identified by their numbers relative to the pro...
Definition problem.cc:15091
LinearAlgebraDistribution *const & dof_distribution_pt() const
Return the pointer to the dof distribution (read-only)
Definition problem.h:1672
void prune_halo_elements_and_nodes(DocInfo &doc_info, const bool &report_stats)
(Irreversibly) prune halo(ed) elements and nodes, usually after another round of refinement,...
Definition problem.cc:1139
virtual void get_inverse_mass_matrix_times_residuals(DoubleVector &Mres)
Return the residual vector multiplied by the inverse mass matrix Virtual so that it can be overloaded...
Definition problem.cc:3665
void check_halo_schemes()
Check the halo/haloed node/element schemes.
Definition problem.h:2245
double Parameter_current
Storage for the present value of the global parameter.
Definition problem.h:755
double *& dof_pt(const unsigned &i)
Pointer to i-th dof in the problem.
Definition problem.h:1829
void assign_eigenvector_to_dofs(DoubleVector &eigenvector)
Assign the eigenvector passed to the function to the dofs in the problem so that it can be output by ...
Definition problem.cc:8896
@ Uniform_matrix_distribution
Definition problem.h:831
@ Default_matrix_distribution
Definition problem.h:829
@ Problem_matrix_distribution
Definition problem.h:830
void send_refinement_info_helper(Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, const unsigned &max_refinement_level_overall, std::map< unsigned, Vector< unsigned > > &refinement_info_for_root_local, Vector< Vector< Vector< unsigned > > > &refinement_info_for_root_elements)
Send refinement information between processors.
Definition problem.cc:18521
unsigned setup_element_count_per_dof()
Function that populates the Element_counter_per_dof vector with the number of elements that contribut...
Definition problem.cc:230
OomphCommunicator * Communicator_pt
The communicator for this problem.
Definition problem.h:1242
double Newton_solver_tolerance
The Tolerance below which the Newton Method is deemed to have converged.
Definition problem.h:596
void set_consistent_pinned_values_for_continuation()
Private helper function that is used to set the appropriate pinned values for continuation.
Definition problem.cc:10659
void activate_bifurcation_tracking(double *const &parameter_pt, const DoubleVector &eigenvector, const bool &block_solve=true)
Activate generic bifurcation tracking for a single (real) eigenvalue where the initial guess for the ...
Definition problem.cc:10322
LinearSolver *& mass_matrix_solver_for_explicit_timestepper_pt()
Return a pointer to the linear solver object used for explicit time stepping.
Definition problem.h:1479
bool Discontinuous_element_formulation
Is the problem a discontinuous one, i.e. can the elemental contributions be treated independently....
Definition problem.h:700
void solve_eigenproblem_legacy(const unsigned &n_eval, Vector< std::complex< double > > &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &steady=true)
Get derivative of an element in the problem wrt a global parameter, used in continuation problems.
Definition problem.cc:8293
bool Doc_imbalance_in_parallel_assembly
Boolean to switch on assessment of load imbalance in parallel assembly of distributed problem.
Definition problem.h:510
long synchronise_eqn_numbers(const bool &assign_local_eqn_numbers=true)
Classify any non-classified nodes into halo/haloed and synchronise equation numbers....
Definition problem.cc:16898
void create_new_linear_algebra_distribution(LinearAlgebraDistribution *&dist_pt)
Get new linear algebra distribution (you're in charge of deleting it!)
Definition problem.cc:299
void p_refine_uniformly_aux(const Vector< unsigned > &nrefine_for_mesh, DocInfo &doc_info, const bool &prune)
Helper function to do compund p-refinement of (all) p-refineable (sub)mesh(es) uniformly as many time...
Definition problem.cc:15785
void calculate_continuation_derivatives_helper(const DoubleVector &z)
A function that performs the guts of the continuation derivative calculation in arc length continuati...
Definition problem.cc:10105
Vector< Problem * > Copy_of_problem_pt
Vector of pointers to copies of the problem used in adaptive bifurcation tracking problems (ALH: TEMP...
Definition problem.h:234
Vector< unsigned > distribute(const Vector< unsigned > &element_partition, DocInfo &doc_info, const bool &report_stats=false)
Distribute the problem and doc, using the specified partition; returns a vector which details the par...
Definition problem.cc:490
Mesh * Mesh_pt
The mesh pointer.
Definition problem.h:167
double Parameter_derivative
Storage for the derivative of the global parameter wrt arc-length.
Definition problem.h:752
TimeStepper *& time_stepper_pt()
Access function for the pointer to the first (presumably only) timestepper.
Definition problem.h:1524
void add_eigenvector_to_dofs(const double &epsilon, const DoubleVector &eigenvector)
Add the eigenvector passed to the function scaled by the constat epsilon to the dofs in the problem s...
Definition problem.cc:8933
Vector< Data * > Global_data_pt
Vector of global data: "Nobody" (i.e. none of the elements etc.) is "in charge" of this Data so it wo...
Definition problem.h:425
void recompute_load_balanced_assembly()
Helper function to re-assign the first and last elements to be assembled by each processor during par...
Definition problem.cc:1775
Vector< double * > Dof_pt
Vector of pointers to dofs.
Definition problem.h:554
void p_adapt()
p-adapt problem: Perform mesh adaptation for (all) refineable (sub)mesh(es), based on their own error...
Definition problem.h:3010
double DTSF_max_increase
Maximum possible increase of dt between time-steps in adaptive schemes.
Definition problem.h:713
bool Must_recompute_load_balance_for_assembly
Boolean indicating that the division of elements over processors during the assembly process must be ...
Definition problem.h:529
virtual void shift_time_values()
Shift all values along to prepare for next timestep.
Definition problem.cc:11827
void set_default_first_and_last_element_for_assembly()
Set default first and last elements for parallel assembly of non-distributed problem.
Definition problem.cc:1699
bool Keep_temporal_error_below_tolerance
Boolean to decide if a timestep is to be rejected if the error estimate post-solve (computed by globa...
Definition problem.h:2338
bool Shut_up_in_newton_solve
Boolean to indicate if all output is suppressed in Problem::newton_solve(). Defaults to false.
Definition problem.h:2320
void set_dofs(const DoubleVector &dofs)
Set the values of the dofs.
Definition problem.cc:3497
void setup_base_mesh_info_after_pruning()
Helper function to re-setup the Base_mesh enumeration (used during load balancing) after pruning.
Definition problem.cc:20295
Vector< Mesh * > Sub_mesh_pt
Vector of pointers to submeshes.
Definition problem.h:170
ExplicitTimeStepper *& explicit_time_stepper_pt()
Return a pointer to the explicit timestepper.
Definition problem.h:1555
Vector< double > Max_res
Maximum residuals at start and after each newton iteration.
Definition problem.h:606
EigenSolver * Eigen_solver_pt
Pointer to the eigen solver for the problem.
Definition problem.h:182
bool Bisect_to_find_bifurcation
Boolean to control wheter bisection is used to located bifurcation.
Definition problem.h:793
void explicit_timestep(const double &dt, const bool &shift_values=true)
Take an explicit timestep of size dt and optionally shift any stored values of the time history.
Definition problem.cc:11111
void delete_all_external_storage()
Wrapper function to delete external storage for each submesh of the problem.
Definition problem.cc:16548
friend class BlockPitchForkLinearSolver
Definition problem.h:159
double DTSF_min_decrease
Minimum allowed decrease of dt between time-steps in adaptive schemes. Lower scaling values will reje...
Definition problem.h:718
virtual void symmetrise_eigenfunction_for_adaptive_pitchfork_tracking()
Virtual function that is used to symmetrise the problem so that the current solution exactly satisfie...
Definition problem.cc:10252
double Minimum_dt_but_still_proceed
If Minimum_dt_but_still_proceed positive, then dt will not be reduced below this value during adaptiv...
Definition problem.h:725
double adaptive_unsteady_newton_solve(const double &dt_desired, const double &epsilon)
Attempt to advance timestep by dt_desired. If the solution fails the timestep will be halved until co...
Definition problem.cc:11249
unsigned Desired_newton_iterations_ds
The desired number of Newton Steps to reach convergence at each step along the arc.
Definition problem.h:784
void rebuild_global_mesh()
If one of the submeshes has changed (e.g. by mesh adaptation) we need to update the global mesh....
Definition problem.cc:1619
void solve_adjoint_eigenproblem(const unsigned &n_eval, Vector< std::complex< double > > &eigenvalue, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an adjoint eigenvalue problem using the same procedure as solve_eigenproblem....
Definition problem.cc:8536
void solve_adjoint_eigenproblem_legacy(const unsigned &n_eval, Vector< std::complex< double > > &eigenvalue, Vector< DoubleVector > &eigenvector, const bool &make_timesteppers_steady=true)
Solve an adjoint eigenvalue problem using the same procedure as solve_eigenproblem....
Definition problem.cc:8479
void activate_hopf_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on Hopf bifurcation tracking using the augmented system specified in the HopfHandler class....
Definition problem.cc:10411
Vector< double * > Halo_dof_pt
Storage for the halo degrees of freedom (only required) when accessing via the global equation number...
Definition problem.h:581
void deactivate_bifurcation_tracking()
Deactivate all bifuraction tracking, by reseting the assembly handler to the default.
Definition problem.h:2482
void globally_convergent_line_search(const Vector< double > &x_old, const double &half_residual_squared_old, DoubleVector &gradient, DoubleVector &newton_dir, double &half_residual_squared, const double &stpmax)
Line search helper for globally convergent Newton method.
Definition problem.cc:9339
void get_hessian_vector_products(DoubleVectorWithHaloEntries const &Y, Vector< DoubleVectorWithHaloEntries > const &C, Vector< DoubleVectorWithHaloEntries > &product)
Return the product of the global hessian (derivative of Jacobian matrix with respect to all variables...
Definition problem.cc:7962
virtual double global_temporal_error_norm()
Function to calculate a global error norm, used in adaptive timestepping to control the change in tim...
Definition problem.h:1230
@ Perform_assembly_using_two_arrays
Definition problem.h:650
@ Perform_assembly_using_maps
Definition problem.h:648
@ Perform_assembly_using_two_vectors
Definition problem.h:647
@ Perform_assembly_using_vectors_of_pairs
Definition problem.h:646
@ Perform_assembly_using_lists
Definition problem.h:649
void refine_distributed_base_mesh(Vector< Vector< Vector< unsigned > > > &to_be_refined_on_each_root, const unsigned &max_level_overall)
Load balance helper routine: refine each new base (sub)mesh based upon the elements to be refined wit...
Definition problem.cc:20183
int Sign_of_jacobian
Storage for the sign of the global Jacobian.
Definition problem.h:745
std::map< GeneralisedElement *, unsigned > Base_mesh_element_number_plus_one
Map which stores the correspondence between a root element and its element number (plus one) within t...
Definition problem.h:538
double Max_residuals
Maximum desired residual: if the maximum residual exceeds this value, the program will exit.
Definition problem.h:610
unsigned nsub_mesh() const
Return number of submeshes.
Definition problem.h:1323
double & time()
Return the current value of continuous time.
Definition problem.cc:11724
unsigned self_test()
Self-test: Check meshes and global data. Return 0 for OK.
Definition problem.cc:13469
unsigned ntime_stepper() const
Return the number of time steppers.
Definition problem.h:1684
void activate_fold_tracking(double *const &parameter_pt, const bool &block_solve=true)
Turn on fold tracking using the augmented system specified in the FoldHandler class....
Definition problem.cc:10296
bool Use_globally_convergent_newton_method
Use the globally convergent newton method.
Definition problem.h:214
LinearSolver * Default_linear_solver_pt
Pointer to the default linear solver.
Definition problem.h:188
double Minimum_ds
Minimum desired value of arc-length.
Definition problem.h:787
void get_data_to_be_sent_during_load_balancing(const Vector< unsigned > &element_domain_on_this_proc, Vector< int > &send_n, Vector< double > &send_data, Vector< int > &send_displacement, Vector< unsigned > &old_domain_for_base_element, Vector< unsigned > &new_domain_for_base_element, unsigned &max_refinement_level_overall)
Load balance helper routine: Get data to be sent to other processors during load balancing and other ...
Definition problem.cc:19472
void restore_dof_values()
Restore the stored values of the degrees of freedom.
Definition problem.cc:8833
double Numerical_zero_for_sparse_assembly
A tolerance used to determine whether the entry in a sparse matrix is zero. If it is then storage nee...
Definition problem.h:671
double FD_step_used_in_get_hessian_vector_products
Definition problem.h:685
virtual void partition_global_mesh(Mesh *&global_mesh_pt, DocInfo &doc_info, Vector< unsigned > &element_domain, const bool &report_stats=false)
Partition the global mesh, return vector specifying the processor number for each element....
Definition problem.cc:964
unsigned Sparse_assemble_with_arrays_initial_allocation
the number of elements to initially allocate for a matrix row within the sparse_assembly_with_two_arr...
Definition problem.h:658
void bifurcation_adapt_doc_errors(const unsigned &bifurcation_type)
A function that is used to document the errors used in the adaptive solution of bifurcation problems.
Definition problem.cc:13841
double Ds_current
Storage for the current step value.
Definition problem.h:780
virtual void set_initial_condition()
Set initial condition (incl previous timesteps). We need to establish this interface because I....
Definition problem.h:1198
LinearSolver * Mass_matrix_solver_for_explicit_timestepper_pt
Pointer to the linear solver used for explicit time steps (this is likely to be different to the line...
Definition problem.h:179
void get_all_halo_data(std::map< unsigned, double * > &map_of_halo_data)
Get pointers to all possible halo data indexed by global equation number in a map.
Definition problem.cc:16570
void load_balance()
Balance the load of a (possibly non-uniformly refined) problem that has already been distributed,...
Definition problem.h:1382
double Minimum_dt
Minimum desired dt: if dt falls below this value, exit.
Definition problem.h:706
double arc_length_step_solve(double *const &parameter_pt, const double &ds, const unsigned &max_adapt=0)
Solve a steady problem using arc-length continuation, when the parameter that becomes a variable corr...
Definition problem.cc:10487
virtual void actions_after_adapt()
Actions that are to be performed after a mesh adaptation.
Definition problem.h:1025
void p_refine_uniformly()
p-refine (all) p-refineable (sub)mesh(es) uniformly and rebuild problem
Definition problem.h:2866
bool Use_continuation_timestepper
Boolean to control original or new storage of dof stuff.
Definition problem.h:758
void calculate_continuation_derivatives_fd(double *const &parameter_pt)
A function to calculate the derivatives with respect to the arc-length required for continuation by f...
Definition problem.cc:10011
LinearAlgebraDistribution * Dof_distribution_pt
The distribution of the DOFs in this problem. This object is created in the Problem constructor and s...
Definition problem.h:460
void refine_uniformly()
Refine (all) refineable (sub)mesh(es) uniformly and rebuild problem.
Definition problem.h:2769
static ContinuationStorageScheme Continuation_time_stepper
Storage for the single static continuation timestorage object.
Definition problem.h:761
bool Problem_is_nonlinear
Boolean flag indicating if we're dealing with a linear or nonlinear Problem – if set to false the New...
Definition problem.h:628
virtual void sparse_assemble_row_or_column_compressed_with_maps(Vector< int * > &column_or_row_index, Vector< int * > &row_or_column_start, Vector< double * > &value, Vector< unsigned > &nnz, Vector< double * > &residual, bool compressed_row_flag)
Private helper function that is used to assemble the Jacobian matrix in the case when the storage is ...
Definition problem.cc:4561
AssemblyHandler * Default_assembly_handler_pt
Pointer to the default assembly handler.
Definition problem.h:194
void get_my_eqns(AssemblyHandler *const &assembly_handler_pt, const unsigned &el_lo, const unsigned &el_hi, Vector< unsigned > &my_eqns)
Helper method that returns the (unique) global equations to which the elements in the range el_lo to ...
Definition problem.cc:6486
virtual void read(std::ifstream &restart_file, bool &unsteady_restart)
Read refinement pattern of all refineable meshes and refine them accordingly, then read all Data and ...
Definition problem.cc:12444
virtual ~Problem()
Virtual destructor to clean up memory.
Definition problem.cc:181
Mesh *& mesh_pt()
Return a pointer to the global mesh.
Definition problem.h:1280
void get_dofs(DoubleVector &dofs) const
Return the vector of dofs, i.e. a vector containing the current values of all unknowns.
Definition problem.cc:2565
Time * Time_pt
Pointer to global time for the problem.
Definition problem.h:197
DoubleVectorWithHaloEntries Element_count_per_dof
Counter that records how many elements contribute to each dof. Used to determine the (discrete) arc-l...
Definition problem.h:560
virtual void actions_before_newton_convergence_check()
Any actions that are to be performed before the residual is checked in the Newton method,...
Definition problem.h:1048
Time *& time_pt()
Return a pointer to the global time object.
Definition problem.h:1504
AssemblyHandler *& assembly_handler_pt()
Return a pointer to the assembly handler object.
Definition problem.h:1570
unsigned newton_solve_continuation(double *const &parameter_pt)
Perform a basic arc-length continuation step using Newton's method. Returns number of Newton steps ta...
Definition problem.cc:9569
double Max_permitted_error_for_halo_check
Threshold for error throwing in Problem::check_halo_schemes()
Definition problem.h:2304
unsigned Sparse_assemble_with_arrays_allocation_increment
the number of elements to add to a matrix row when the initial allocation is exceeded within the spar...
Definition problem.h:663
void reset_assembly_handler_to_default()
Reset the system to the standard non-augemented state.
Definition problem.cc:10468
void solve_eigenproblem(const unsigned &n_eval, Vector< std::complex< double > > &alpha, Vector< double > &beta, Vector< DoubleVector > &eigenvector_real, Vector< DoubleVector > &eigenvector_imag, const bool &steady=true)
Solve an eigenproblem as assembled by the Problem's constituent elements. Calculate (at least) n_eval...
Definition problem.cc:8348
virtual void actions_after_newton_step()
Any actions that are to be performed after each individual Newton step. Most likely to be used for di...
Definition problem.h:1058
bool Pause_at_end_of_sparse_assembly
Protected boolean flag to halt program execution during sparse assemble process to assess peak memory...
Definition problem.h:633
void remove_null_pointers_from_external_halo_node_storage()
Consolidate external halo node storage by removing nulled out pointers in external halo and haloed sc...
Definition problem.cc:3264
Vector< double > * Saved_dof_pt
Pointer to vector for backup of dofs.
Definition problem.h:207
void unsteady_newton_solve(const double &dt)
Advance time by dt and solve by Newton's method. This version always shifts time values.
Definition problem.cc:11146
AssemblyHandler * Assembly_handler_pt
Definition problem.h:185
virtual void actions_before_read_unstructured_meshes()
Actions that are to be performed before reading in restart data for problems involving unstructured b...
Definition problem.h:1095
Vector< TimeStepper * > Time_stepper_pt
The Vector of time steppers (there could be many different ones in multiphysics problems)
Definition problem.h:201
void doc_errors()
Get max and min error for all elements in submeshes.
Definition problem.h:3055
bool Mass_matrix_reuse_is_enabled
Is re-use of the mass matrix in explicit timestepping enabled Default:false.
Definition problem.h:691
void get_derivative_wrt_global_parameter(double *const &parameter_pt, DoubleVector &result)
Get the derivative of the entire residuals vector wrt a global parameter, used in continuation proble...
Definition problem.cc:7864
bool distributed() const
If we have MPI return the "problem has been distributed" flag, otherwise it can't be distributed so r...
Definition problem.h:808
bool are_hessian_products_calculated_analytically()
Function to determine whether the hessian products are calculated analytically.
Definition problem.h:303
bool does_pointer_correspond_to_problem_data(double *const &parameter_pt)
Return a boolean flag to indicate whether the pointer parameter_pt refers to values stored in a Data ...
Definition problem.cc:10039
ExplicitTimeStepper * Explicit_time_stepper_pt
Pointer to a single explicit timestepper.
Definition problem.h:204
double arc_length_step_solve_helper(double *const &parameter_pt, const double &ds, const unsigned &max_adapt)
Private helper function that actually contains the guts of the arc-length stepping,...
Definition problem.cc:10708
bool Use_predictor_values_as_initial_guess
Use values from the time stepper predictor as an initial guess.
Definition problem.h:229
RefineableElements are FiniteElements that may be subdivided into children to provide a better local ...
Base class for refineable meshes. Provides standardised interfaces for the following standard mesh ad...
General SolidMesh class.
Definition mesh.h:2562
A Class for nodes that deform elastically (i.e. position is an unknown in the problem)....
Definition nodes.h:1686
/////////////////////////////////////////////////////////////////////// /////////////////////////////...
Definition spines.h:613
//////////////////////////////////////////////////////////////////////////// ////////////////////////...
//////////////////////////////////////////////////////////////////////
void output(std::ostream &outfile)
Output function: x,y,u or x,y,z,u.
////////////////////////////////////////////////////////////////////// //////////////////////////////...
virtual unsigned ndt() const =0
Number of timestep increments that are required by the scheme.
virtual void shift_time_values(Data *const &data_pt)=0
This function advances the Data's time history so that we can move on to the next timestep.
virtual void set_weights()=0
Function to set the weights for present timestep (don't need to pass present timestep or previous tim...
ExplicitTimeStepper * explicit_predictor_pt()
Get the pointer to the explicit timestepper to use as a predictor in adaptivity if Predict_by_explici...
virtual void calculate_predicted_values(Data *const &data_pt)
Do the predictor step for data stored in a Data object (currently empty – overwrite for specific sche...
virtual void set_predictor_weights()
Set the weights for the predictor previous timestep (currently empty – overwrite for specific scheme)
virtual void actions_before_timestep(Problem *problem_pt)
Interface for any actions that need to be performed before a time step.
virtual void actions_after_timestep(Problem *problem_pt)
Interface for any actions that need to be performed after a time step.
virtual void assign_initial_values_impulsive(Data *const &data_pt)=0
Initialise the time-history for the Data values corresponding to an impulsive start.
void make_steady()
Function to make the time stepper temporarily steady. This is trivially achieved by setting all the w...
virtual void undo_make_steady()
Reset the is_steady status of a specific TimeStepper to its default and re-assign the weights.
void update_predicted_time(const double &new_time)
Set the time that the current predictions apply for, only needed for paranoid checks when doing Predi...
Time *const & time_pt() const
Access function for the pointer to time (const version)
bool is_steady() const
Flag to indicate if a timestepper has been made steady (possibly temporarily to switch off time-depen...
virtual void set_error_weights()
Set the weights for the error computation, (currently empty – overwrite for specific scheme)
Class to keep track of discrete/continous time. It is essential to have a single Time object when usi...
double & time()
Return the current value of the continuous time.
void shift_dt()
Update all stored values of dt by shifting each value along the array. This function must be called b...
void initialise_dt(const double &dt_)
Set all timesteps to the same value, dt.
double & dt(const unsigned &t=0)
Return the value of the t-th stored timestep (t=0: present; t>0: previous).
unsigned ndt() const
Return the number of timesteps stored.
void resize(const unsigned &n_dt)
Resize the vector holding the number of previous timesteps and initialise the new values to zero.
///////////////////////////////////////////////////////////////// ///////////////////////////////////...
TreeRoot is a Tree that forms the root of a (recursive) tree. The "root node" is special as it holds ...
Definition tree.h:324
Base class for triangle meshes (meshes made of 2D triangle elements). Note: we choose to template Tri...
A slight extension to the standard template vector class so that we can include "graceful" array rang...
Definition Vector.h:58
bool Doc_comprehensive_timings
Global boolean to switch on comprehensive timing – can probably be declared const false when developm...
void partition_distributed_mesh(Problem *problem_pt, const unsigned &objective, Vector< unsigned > &element_domain_on_this_proc, const bool &bypass_metis=false)
Use METIS to assign each element in an already-distributed mesh to a domain. On return,...
void partition_mesh(Problem *problem_pt, const unsigned &ndomain, const unsigned &objective, Vector< unsigned > &element_domain)
Use METIS to assign each element to a domain. On return, element_domain[ielem] contains the number of...
std::string to_string(T object, unsigned float_precision=8)
Conversion function that should work for anything with operator<< defined (at least all basic types).
void clean_up_memory()
Clean up function that deletes anything dynamically allocated in this namespace.
void setup()
Setup terminate helper.
double timer()
returns the time in seconds after some point in past
std::string convert_secs_to_formatted_string(const double &time_in_sec)
Returns a nicely formatted string from an input time in seconds; the format depends on the size of ti...
//////////////////////////////////////////////////////////////////// ////////////////////////////////...
void pause(std::string message)
Pause and display message.
OomphInfo oomph_info
Single (global) instantiation of the OomphInfo object – this is used throughout the library as a "rep...